From d4710a37fcc48701aa7cca7c365a3fee60c47f0e Mon Sep 17 00:00:00 2001 From: criamico Date: Wed, 13 Mar 2024 15:49:16 +0100 Subject: [PATCH 01/38] [Fleet] Implement state machine behavior for package install --- .../integrations_state_machine.test.ts | 68 ++++++++++ .../packages/integrations_state_machine.ts | 122 ++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts new file mode 100644 index 0000000000000..919b4bc5b4cd2 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { handleStateMachine } from './integrations_state_machine'; + +const getTestStates = (mockEvent1: any, mockEvent2: any, mockEvent3: any) => { + return { + state1: { + event: mockEvent1, + nextState: 'state2', + }, + state2: { + event: mockEvent2, + nextState: 'state3', + }, + state3: { + event: mockEvent3, + nextState: 'end', + }, + }; +}; + +describe('handleStateMachine', () => { + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should execute all the state machine transitions based on the provided data structure', async () => { + const mockEventState1 = jest.fn(); + const mockEventState2 = jest.fn(); + const mockEventState3 = jest.fn(); + const testStates = getTestStates(mockEventState1, mockEventState2, mockEventState3); + + await handleStateMachine('state1', testStates, undefined); + expect(mockEventState1).toHaveBeenCalledTimes(1); + expect(mockEventState2).toHaveBeenCalledTimes(1); + expect(mockEventState3).toHaveBeenCalledTimes(1); + }); + + it('should execute the transition from the provided state', async () => { + const mockEventState1 = jest.fn(); + const mockEventState2 = jest.fn(); + const mockEventState3 = jest.fn(); + const testStates = getTestStates(mockEventState1, mockEventState2, mockEventState3); + await handleStateMachine('state2', testStates, undefined); + + expect(mockEventState1).toHaveBeenCalledTimes(0); + expect(mockEventState2).toHaveBeenCalledTimes(1); + expect(mockEventState3).toHaveBeenCalledTimes(1); + }); + + it('should exit when a state returns error', async () => { + const error = new Error('Insallation failed'); + const mockEventState1 = jest.fn().mockRejectedValue(error); + const mockEventState2 = jest.fn(); + const mockEventState3 = jest.fn(); + const testStates = getTestStates(mockEventState1, mockEventState2, mockEventState3); + await handleStateMachine('state1', testStates, undefined); + + expect(mockEventState1).toHaveBeenCalledTimes(1); + expect(mockEventState2).toHaveBeenCalledTimes(0); + expect(mockEventState3).toHaveBeenCalledTimes(0); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts new file mode 100644 index 0000000000000..aa1e4408e705a --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +// import { appContextService } from '../../app_context'; + +export interface State { + event: any; + nextState?: string; + currentStatus?: string; +} +export type StatusName = 'success' | 'failed' | 'pending'; + +export const installStateNames = [ + 'create_restart_installation', + 'install_kibana_assets', + 'install_ml_model', + 'install_index_template_pipelines', + 'remove_legacy_templates', + 'update_current_write_indices', + 'install_transforms', + 'delete_previous_pipelines', + 'save_archive_entries_from_assets_map', + 'update_so', +] as const; + +type StateNamesTuple = typeof installStateNames; +type StateNames = StateNamesTuple[number]; + +// example our install states +const installStates: Record = { + create_restart_installation: { + nextState: 'install_kibana_assets', + event: () => undefined, + }, + install_kibana_assets: { + event: () => undefined, + nextState: 'install_ml_model', + }, + install_ml_model: { + event: () => undefined, + nextState: 'install_index_template_pipelines', + }, + install_index_template_pipelines: { + event: () => undefined, + nextState: 'remove_legacy_templates', + }, + remove_legacy_templates: { + event: () => undefined, + nextState: 'update_current_write_indices', + }, + update_current_write_indices: { + event: () => undefined, + nextState: 'install_transforms', + }, + install_transforms: { + event: () => undefined, + nextState: 'delete_previous_pipelines', + }, + delete_previous_pipelines: { + event: () => undefined, + nextState: 'save_archive_entries_from_assets_map', + }, + save_archive_entries_from_assets_map: { + event: () => undefined, + nextState: 'update_so', + }, + update_so: { + event: () => undefined, + nextState: 'end', + }, +}; + +export async function handleStateMachine( + startState: StateNames, + states: Record, + data: any +) { + await handleTransition(startState, states, data); +} + +async function handleTransition( + currentStateName: StateNames, + states: Record, + stateData?: any +) { + // const logger = appContextService.getLogger(); + const currentState = states[currentStateName]; + let currentStatus = 'pending'; + let stateResult; + + if (typeof currentState.event === 'function') { + try { + stateResult = stateData ? await currentState.event(...stateData) : await currentState.event(); + currentStatus = 'success'; + } catch (error) { + currentStatus = 'failed'; + console.error( + `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}` + ); + // logger.warn( + // `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}` + // ); + } + } else { + currentStatus = 'failed'; + } + // update SO with current state data + console.log( + `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` + ); + // logger.info( + // `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` + // ); + if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { + handleTransition(currentState.nextState, states, stateData); + } else { + return; + } +} From 94aaa4d47c0d18313203ebd930876c944e36b28a Mon Sep 17 00:00:00 2001 From: criamico Date: Wed, 13 Mar 2024 16:56:43 +0100 Subject: [PATCH 02/38] fix types and test --- .../integrations_state_machine.test.ts | 24 +++++++++++++++++- .../packages/integrations_state_machine.ts | 25 ++++++++----------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index 919b4bc5b4cd2..b1d7587131dec 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -5,6 +5,9 @@ * 2.0. */ +import { createAppContextStartContractMock } from '../../../mocks'; +import { appContextService } from '../..'; + import { handleStateMachine } from './integrations_state_machine'; const getTestStates = (mockEvent1: any, mockEvent2: any, mockEvent3: any) => { @@ -25,8 +28,15 @@ const getTestStates = (mockEvent1: any, mockEvent2: any, mockEvent3: any) => { }; describe('handleStateMachine', () => { + let mockContract: ReturnType; + beforeEach(async () => { + // prevents `Logger not set.` and other appContext errors + mockContract = createAppContextStartContractMock(); + appContextService.start(mockContract); + }); afterEach(() => { jest.resetAllMocks(); + appContextService.stop(); }); it('should execute all the state machine transitions based on the provided data structure', async () => { @@ -39,6 +49,15 @@ describe('handleStateMachine', () => { expect(mockEventState1).toHaveBeenCalledTimes(1); expect(mockEventState2).toHaveBeenCalledTimes(1); expect(mockEventState3).toHaveBeenCalledTimes(1); + expect(mockContract.logger?.info).toHaveBeenCalledWith( + 'state: state1 - status success - stateResult: undefined - nextState: state2' + ); + expect(mockContract.logger?.info).toHaveBeenCalledWith( + 'state: state2 - status success - stateResult: undefined - nextState: state3' + ); + expect(mockContract.logger?.info).toHaveBeenCalledWith( + 'state: state3 - status success - stateResult: undefined - nextState: end' + ); }); it('should execute the transition from the provided state', async () => { @@ -54,7 +73,7 @@ describe('handleStateMachine', () => { }); it('should exit when a state returns error', async () => { - const error = new Error('Insallation failed'); + const error = new Error('Installation failed'); const mockEventState1 = jest.fn().mockRejectedValue(error); const mockEventState2 = jest.fn(); const mockEventState3 = jest.fn(); @@ -64,5 +83,8 @@ describe('handleStateMachine', () => { expect(mockEventState1).toHaveBeenCalledTimes(1); expect(mockEventState2).toHaveBeenCalledTimes(0); expect(mockEventState3).toHaveBeenCalledTimes(0); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Error during execution of state "state1" with status "failed": Installation failed' + ); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index aa1e4408e705a..acc5b2c70a500 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -// import { appContextService } from '../../app_context'; +import { appContextService } from '../../app_context'; export interface State { event: any; @@ -28,9 +28,10 @@ export const installStateNames = [ type StateNamesTuple = typeof installStateNames; type StateNames = StateNamesTuple[number]; +type StateMachineStates = Record; // example our install states -const installStates: Record = { +const installStates: StateMachineStates = { create_restart_installation: { nextState: 'install_kibana_assets', event: () => undefined, @@ -74,19 +75,19 @@ const installStates: Record = { }; export async function handleStateMachine( - startState: StateNames, - states: Record, + startState: string, + states: StateMachineStates, data: any ) { await handleTransition(startState, states, data); } async function handleTransition( - currentStateName: StateNames, - states: Record, + currentStateName: string, + states: StateMachineStates, stateData?: any ) { - // const logger = appContextService.getLogger(); + const logger = appContextService.getLogger(); const currentState = states[currentStateName]; let currentStatus = 'pending'; let stateResult; @@ -97,23 +98,17 @@ async function handleTransition( currentStatus = 'success'; } catch (error) { currentStatus = 'failed'; - console.error( + logger.warn( `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}` ); - // logger.warn( - // `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}` - // ); } } else { currentStatus = 'failed'; } // update SO with current state data - console.log( + logger.info( `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` ); - // logger.info( - // `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` - // ); if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { handleTransition(currentState.nextState, states, stateData); } else { From b89756e1ece6757d8d22796c3a2580d36db4bc84 Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 14 Mar 2024 15:18:47 +0100 Subject: [PATCH 03/38] Improve naming --- .../integrations_state_machine.test.ts | 12 +++--- .../packages/integrations_state_machine.ts | 40 ++++++++++--------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index b1d7587131dec..830661c4f8528 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -13,15 +13,15 @@ import { handleStateMachine } from './integrations_state_machine'; const getTestStates = (mockEvent1: any, mockEvent2: any, mockEvent3: any) => { return { state1: { - event: mockEvent1, + onTransitionTo: mockEvent1, nextState: 'state2', }, state2: { - event: mockEvent2, + onTransitionTo: mockEvent2, nextState: 'state3', }, state3: { - event: mockEvent3, + onTransitionTo: mockEvent3, nextState: 'end', }, }; @@ -49,13 +49,13 @@ describe('handleStateMachine', () => { expect(mockEventState1).toHaveBeenCalledTimes(1); expect(mockEventState2).toHaveBeenCalledTimes(1); expect(mockEventState3).toHaveBeenCalledTimes(1); - expect(mockContract.logger?.info).toHaveBeenCalledWith( + expect(mockContract.logger?.debug).toHaveBeenCalledWith( 'state: state1 - status success - stateResult: undefined - nextState: state2' ); - expect(mockContract.logger?.info).toHaveBeenCalledWith( + expect(mockContract.logger?.debug).toHaveBeenCalledWith( 'state: state2 - status success - stateResult: undefined - nextState: state3' ); - expect(mockContract.logger?.info).toHaveBeenCalledWith( + expect(mockContract.logger?.debug).toHaveBeenCalledWith( 'state: state3 - status success - stateResult: undefined - nextState: end' ); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index acc5b2c70a500..3c8d3bec1f733 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -7,7 +7,7 @@ import { appContextService } from '../../app_context'; export interface State { - event: any; + onTransitionTo: any; nextState?: string; currentStatus?: string; } @@ -34,42 +34,42 @@ type StateMachineStates = Record; const installStates: StateMachineStates = { create_restart_installation: { nextState: 'install_kibana_assets', - event: () => undefined, + onTransitionTo: () => undefined, }, install_kibana_assets: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'install_ml_model', }, install_ml_model: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'install_index_template_pipelines', }, install_index_template_pipelines: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'remove_legacy_templates', }, remove_legacy_templates: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'update_current_write_indices', }, update_current_write_indices: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'install_transforms', }, install_transforms: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'delete_previous_pipelines', }, delete_previous_pipelines: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'save_archive_entries_from_assets_map', }, save_archive_entries_from_assets_map: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'update_so', }, update_so: { - event: () => undefined, + onTransitionTo: () => undefined, nextState: 'end', }, }; @@ -77,24 +77,26 @@ const installStates: StateMachineStates = { export async function handleStateMachine( startState: string, states: StateMachineStates, - data: any + context: any // TODO: find better type for this ) { - await handleTransition(startState, states, data); + await handleState(startState, states, context); } -async function handleTransition( +async function handleState( currentStateName: string, states: StateMachineStates, - stateData?: any + context?: any ) { const logger = appContextService.getLogger(); const currentState = states[currentStateName]; let currentStatus = 'pending'; let stateResult; - if (typeof currentState.event === 'function') { + if (typeof currentState.onTransitionTo === 'function') { try { - stateResult = stateData ? await currentState.event(...stateData) : await currentState.event(); + stateResult = context + ? await currentState.onTransitionTo(...context) + : await currentState.onTransitionTo(); currentStatus = 'success'; } catch (error) { currentStatus = 'failed'; @@ -106,11 +108,11 @@ async function handleTransition( currentStatus = 'failed'; } // update SO with current state data - logger.info( + logger.debug( `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` ); if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { - handleTransition(currentState.nextState, states, stateData); + handleState(currentState.nextState, states, context); } else { return; } From 7e027eca6d01ccb0331be525e88b2b6e0c0c691a Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 14 Mar 2024 17:36:17 +0100 Subject: [PATCH 04/38] Add context and start writing new function --- .../services/epm/packages/_install_package.ts | 81 +++------- .../_state_machine_package_install.ts | 145 ++++++++++++++++++ .../server/services/epm/packages/install.ts | 77 ++++++++++ .../integrations_state_machine.test.ts | 67 +++++--- .../packages/integrations_state_machine.ts | 86 ++--------- 5 files changed, 305 insertions(+), 151 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 8b8a44b55e222..f68cef6ccd31f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -21,7 +21,6 @@ import type { PackageInstallContext } from '../../../../common/types'; import { getNormalizedDataStreams } from '../../../../common/services'; import { - MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT, @@ -45,12 +44,12 @@ import { installTransforms } from '../elasticsearch/transform/install'; import { installMlModel } from '../elasticsearch/ml_model'; import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; import { saveArchiveEntriesFromAssetsMap } from '../archive/storage'; -import { ConcurrentInstallOperationError, PackageSavedObjectConflictError } from '../../../errors'; +import { PackageSavedObjectConflictError } from '../../../errors'; import { appContextService, packagePolicyService } from '../..'; import { auditLoggingService } from '../../audit_logging'; -import { createInstallation, restartInstallation } from './install'; +import { createRestartInstallation } from './install'; import { withPackageSpan } from './utils'; import { clearLatestFailedAttempts } from './install_errors_helpers'; import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; @@ -97,61 +96,18 @@ export async function _installPackage({ const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo; try { - // if some installation already exists - if (installedPkg) { - const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; - const hasExceededTimeout = - Date.now() - Date.parse(installedPkg.attributes.install_started_at) < - MAX_TIME_COMPLETE_INSTALL; - logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); - - // if the installation is currently running, don't try to install - // instead, only return already installed assets - if (isStatusInstalling && hasExceededTimeout) { - // If this is a forced installation, ignore the timeout and restart the installation anyway - logger.debug(`Package install - Installation is running and has exceeded timeout`); - - if (force) { - logger.debug(`Package install - Forced installation, restarting`); - await restartInstallation({ - savedObjectsClient, - pkgName, - pkgVersion, - installSource, - verificationResult, - }); - } else { - throw new ConcurrentInstallOperationError( - `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ - pkgVersion || 'unknown' - } detected, aborting.` - ); - } - } else { - // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL - // (it might be stuck) update the saved object and proceed - logger.debug( - `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` - ); - await restartInstallation({ - savedObjectsClient, - pkgName, - pkgVersion, - installSource, - verificationResult, - }); - } - } else { - logger.debug(`Package install - Create installation`); - await createInstallation({ - savedObjectsClient, - packageInfo, - installSource, - spaceId, - verificationResult, - }); - } + await createRestartInstallation({ + savedObjectsClient, + logger, + packageInfo, + installSource, + spaceId, + force, + verificationResult, + installedPkg, + }); logger.debug(`Package install - Installing Kibana assets`); + // step install_kibana_assets const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => installKibanaAssetsAndReferences({ savedObjectsClient, @@ -197,7 +153,7 @@ export async function _installPackage({ ) )); } - + // step install_ml_model // installs ml models logger.debug(`Package install - installing ML models`); esReferences = await withPackageSpan('Install ML models', () => @@ -206,6 +162,8 @@ export async function _installPackage({ let indexTemplates: IndexTemplateEntry[] = []; + // step install_index_template_pipelines + // it should contain the case for integration and the one for inputs if (packageInfo.type === 'integration') { logger.debug( `Package install - Installing index templates and pipelines, packageInfo.type ${packageInfo.type}` @@ -255,7 +213,7 @@ export async function _installPackage({ indexTemplates = installedTemplates; } } - + // step remove_legacy_templates try { logger.debug(`Package install - Removing legacy templates`); await removeLegacyTemplates({ packageInfo, esClient, logger }); @@ -263,6 +221,7 @@ export async function _installPackage({ logger.warn(`Error removing legacy templates: ${e.message}`); } + // step update_current_write_indices // update current backing indices of each data stream logger.debug(`Package install - Updating backing indices of each data stream`); await withPackageSpan('Update write indices', () => @@ -272,6 +231,7 @@ export async function _installPackage({ }) ); logger.debug(`Package install - Installing transforms`); + // steps install_transforms ({ esReferences } = await withPackageSpan('Install transforms', () => installTransforms({ packageInstallContext, @@ -287,6 +247,7 @@ export async function _installPackage({ // If this is an update or retrying an update, delete the previous version's pipelines // Top-level pipeline assets will not be removed on upgrade as of ml model package addition which requires previous // assets to remain installed. This is a temporary solution - more robust solution tracked here https://github.com/elastic/kibana/issues/115035 + // steps delete_previous_pipelines - should contain the two ifs if ( paths.filter((path) => isTopLevelPipeline(path)).length === 0 && (installType === 'update' || installType === 'reupdate') && @@ -322,6 +283,7 @@ export async function _installPackage({ } const installedKibanaAssetsRefs = await kibanaAssetPromise; + // step save_archive_entries_from_assets_map logger.debug(`Package install - Updating archive entries`); const packageAssetResults = await withPackageSpan('Update archive entries', () => saveArchiveEntriesFromAssetsMap({ @@ -344,6 +306,7 @@ export async function _installPackage({ id: pkgName, savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, }); + // step update_SO logger.debug(`Package install - Updating install status`); await withPackageSpan('Update install status', () => savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts new file mode 100644 index 0000000000000..fe90cef2bdba2 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + ElasticsearchClient, + Logger, + SavedObject, + SavedObjectsClientContract, + ISavedObjectsImporter, +} from '@kbn/core/server'; + +import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server'; + +import type { HTTPAuthorizationHeader } from '../../../../common/http_authorization_header'; +import type { PackageInstallContext } from '../../../../common/types'; + +import type { + Installation, + InstallType, + InstallSource, + PackageVerificationResult, +} from '../../../types'; + +import { createRestartInstallation } from './install'; +import type { StateMachineDefinition } from './integrations_state_machine'; +import { handleStateMachine } from './integrations_state_machine'; + +export const installStateNames = [ + 'create_restart_installation', + 'install_kibana_assets', + 'install_ml_model', + 'install_index_template_pipelines', + 'remove_legacy_templates', + 'update_current_write_indices', + 'install_transforms', + 'delete_previous_pipelines', + 'save_archive_entries_from_assets_map', + 'update_so', +] as const; + +type StateNamesTuple = typeof installStateNames; +type StateNames = StateNamesTuple[number]; + +export async function _stateMachineInstallPackage({ + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + esClient, + logger, + installedPkg, + packageInstallContext, + installType, + installSource, + spaceId, + force, + verificationResult, + authorizationHeader, + ignoreMappingUpdateErrors, + skipDataStreamRollover, +}: { + savedObjectsClient: SavedObjectsClientContract; + savedObjectsImporter: Pick; + savedObjectTagAssignmentService: IAssignmentService; + savedObjectTagClient: ITagsClient; + esClient: ElasticsearchClient; + logger: Logger; + installedPkg?: SavedObject; + packageInstallContext: PackageInstallContext; + installType: InstallType; + installSource: InstallSource; + spaceId: string; + force?: boolean; + verificationResult?: PackageVerificationResult; + authorizationHeader?: HTTPAuthorizationHeader | null; + ignoreMappingUpdateErrors?: boolean; + skipDataStreamRollover?: boolean; +}) { + const { packageInfo, paths } = packageInstallContext; + const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo; + + // our install states + const installStates: StateMachineDefinition = { + context: { + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + pkgName, + pkgTitle, + packageInstallContext, + paths, + installedPkg, + logger, + spaceId, + assetTags: packageInfo?.asset_tags, + }, + states: { + create_restart_installation: { + nextState: 'install_kibana_assets', + onTransitionTo: createRestartInstallation, + }, + install_kibana_assets: { + onTransitionTo: () => undefined, + nextState: 'install_ml_model', + }, + install_ml_model: { + onTransitionTo: () => undefined, + nextState: 'install_index_template_pipelines', + }, + install_index_template_pipelines: { + onTransitionTo: () => undefined, + nextState: 'remove_legacy_templates', + }, + remove_legacy_templates: { + onTransitionTo: () => undefined, + nextState: 'update_current_write_indices', + }, + update_current_write_indices: { + onTransitionTo: () => undefined, + nextState: 'install_transforms', + }, + install_transforms: { + onTransitionTo: () => undefined, + nextState: 'delete_previous_pipelines', + }, + delete_previous_pipelines: { + onTransitionTo: () => undefined, + nextState: 'save_archive_entries_from_assets_map', + }, + save_archive_entries_from_assets_map: { + onTransitionTo: () => undefined, + nextState: 'update_so', + }, + update_so: { + onTransitionTo: () => undefined, + nextState: 'end', + }, + }, + }; + await handleStateMachine('create_restart_installation', installStates); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 43b0c9d68a04c..bcef62620492b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -971,6 +971,83 @@ export const updateInstallStatusToFailed = async ({ } }; +export async function createRestartInstallation({ + savedObjectsClient, + logger, + packageInfo, + installSource, + spaceId, + force, + verificationResult, + installedPkg, +}: { + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; + packageInfo: InstallablePackage; + installSource: InstallSource; + spaceId: string; + force?: boolean; + verificationResult?: PackageVerificationResult; + installedPkg?: SavedObject; +}) { + const { name: pkgName, version: pkgVersion } = packageInfo; + // if some installation already exists + if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); + + // if the installation is currently running, don't try to install + // instead, only return already installed assets + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + logger.debug(`Package install - Installation is running and has exceeded timeout`); + + if (force) { + logger.debug(`Package install - Forced installation, restarting`); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } + } else { + // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL + // (it might be stuck) update the saved object and proceed + logger.debug( + `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` + ); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } + } else { + logger.debug(`Package install - Create installation`); + // step create_installation + await createInstallation({ + savedObjectsClient, + packageInfo, + installSource, + spaceId, + verificationResult, + }); + } +} + export async function restartInstallation(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index 830661c4f8528..0ca5de89b0015 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -10,19 +10,22 @@ import { appContextService } from '../..'; import { handleStateMachine } from './integrations_state_machine'; -const getTestStates = (mockEvent1: any, mockEvent2: any, mockEvent3: any) => { +const getTestDefinition = (mockEvent1: any, mockEvent2: any, mockEvent3: any, context?: any) => { return { - state1: { - onTransitionTo: mockEvent1, - nextState: 'state2', - }, - state2: { - onTransitionTo: mockEvent2, - nextState: 'state3', - }, - state3: { - onTransitionTo: mockEvent3, - nextState: 'end', + context, + states: { + state1: { + onTransitionTo: mockEvent1, + nextState: 'state2', + }, + state2: { + onTransitionTo: mockEvent2, + nextState: 'state3', + }, + state3: { + onTransitionTo: mockEvent3, + nextState: 'end', + }, }, }; }; @@ -43,9 +46,9 @@ describe('handleStateMachine', () => { const mockEventState1 = jest.fn(); const mockEventState2 = jest.fn(); const mockEventState3 = jest.fn(); - const testStates = getTestStates(mockEventState1, mockEventState2, mockEventState3); + const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); - await handleStateMachine('state1', testStates, undefined); + await handleStateMachine('state1', testDefinition); expect(mockEventState1).toHaveBeenCalledTimes(1); expect(mockEventState2).toHaveBeenCalledTimes(1); expect(mockEventState3).toHaveBeenCalledTimes(1); @@ -64,8 +67,8 @@ describe('handleStateMachine', () => { const mockEventState1 = jest.fn(); const mockEventState2 = jest.fn(); const mockEventState3 = jest.fn(); - const testStates = getTestStates(mockEventState1, mockEventState2, mockEventState3); - await handleStateMachine('state2', testStates, undefined); + const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); + await handleStateMachine('state2', testDefinition); expect(mockEventState1).toHaveBeenCalledTimes(0); expect(mockEventState2).toHaveBeenCalledTimes(1); @@ -77,8 +80,8 @@ describe('handleStateMachine', () => { const mockEventState1 = jest.fn().mockRejectedValue(error); const mockEventState2 = jest.fn(); const mockEventState3 = jest.fn(); - const testStates = getTestStates(mockEventState1, mockEventState2, mockEventState3); - await handleStateMachine('state1', testStates, undefined); + const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); + await handleStateMachine('state1', testDefinition); expect(mockEventState1).toHaveBeenCalledTimes(1); expect(mockEventState2).toHaveBeenCalledTimes(0); @@ -87,4 +90,32 @@ describe('handleStateMachine', () => { 'Error during execution of state "state1" with status "failed": Installation failed' ); }); + + it('should call the onTransition function with the provided data', async () => { + const mockEventState1 = jest.fn(); + const mockEventState2 = jest.fn(); + const mockEventState3 = jest.fn(); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockEventState1, + mockEventState2, + mockEventState3, + contextData + ); + + await handleStateMachine('state1', testDefinition); + expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockEventState2).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockEventState3).toHaveBeenCalledWith({ testData: 'test' }); + + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'state: state1 - status success - stateResult: undefined - nextState: state2' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'state: state2 - status success - stateResult: undefined - nextState: state3' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'state: state3 - status success - stateResult: undefined - nextState: end' + ); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 3c8d3bec1f733..b79b1f83ce937 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -5,98 +5,36 @@ * 2.0. */ import { appContextService } from '../../app_context'; - export interface State { onTransitionTo: any; nextState?: string; currentStatus?: string; } -export type StatusName = 'success' | 'failed' | 'pending'; - -export const installStateNames = [ - 'create_restart_installation', - 'install_kibana_assets', - 'install_ml_model', - 'install_index_template_pipelines', - 'remove_legacy_templates', - 'update_current_write_indices', - 'install_transforms', - 'delete_previous_pipelines', - 'save_archive_entries_from_assets_map', - 'update_so', -] as const; - -type StateNamesTuple = typeof installStateNames; -type StateNames = StateNamesTuple[number]; -type StateMachineStates = Record; -// example our install states -const installStates: StateMachineStates = { - create_restart_installation: { - nextState: 'install_kibana_assets', - onTransitionTo: () => undefined, - }, - install_kibana_assets: { - onTransitionTo: () => undefined, - nextState: 'install_ml_model', - }, - install_ml_model: { - onTransitionTo: () => undefined, - nextState: 'install_index_template_pipelines', - }, - install_index_template_pipelines: { - onTransitionTo: () => undefined, - nextState: 'remove_legacy_templates', - }, - remove_legacy_templates: { - onTransitionTo: () => undefined, - nextState: 'update_current_write_indices', - }, - update_current_write_indices: { - onTransitionTo: () => undefined, - nextState: 'install_transforms', - }, - install_transforms: { - onTransitionTo: () => undefined, - nextState: 'delete_previous_pipelines', - }, - delete_previous_pipelines: { - onTransitionTo: () => undefined, - nextState: 'save_archive_entries_from_assets_map', - }, - save_archive_entries_from_assets_map: { - onTransitionTo: () => undefined, - nextState: 'update_so', - }, - update_so: { - onTransitionTo: () => undefined, - nextState: 'end', - }, -}; +export type StatusName = 'success' | 'failed' | 'pending'; +export type StateMachineStates = Record; +export interface StateMachineDefinition { + context?: any; + states: StateMachineStates; +} export async function handleStateMachine( startState: string, - states: StateMachineStates, - context: any // TODO: find better type for this + definition: StateMachineDefinition ) { - await handleState(startState, states, context); + await handleState(startState, definition); } -async function handleState( - currentStateName: string, - states: StateMachineStates, - context?: any -) { +async function handleState(currentStateName: string, definition: StateMachineDefinition) { const logger = appContextService.getLogger(); + const { states, context } = definition; const currentState = states[currentStateName]; let currentStatus = 'pending'; let stateResult; if (typeof currentState.onTransitionTo === 'function') { try { - stateResult = context - ? await currentState.onTransitionTo(...context) - : await currentState.onTransitionTo(); + stateResult = await currentState.onTransitionTo.call(undefined, context); currentStatus = 'success'; } catch (error) { currentStatus = 'failed'; @@ -112,7 +50,7 @@ async function handleState( `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` ); if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { - handleState(currentState.nextState, states, context); + handleState(currentState.nextState, definition); } else { return; } From f68cdcdf62338b1641158ab6f0553e957649ef54 Mon Sep 17 00:00:00 2001 From: criamico Date: Mon, 18 Mar 2024 11:44:37 +0100 Subject: [PATCH 05/38] Add post transition function --- .../integrations_state_machine.test.ts | 36 ++++++++++++++++++- .../packages/integrations_state_machine.ts | 10 +++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index 0ca5de89b0015..574c849e8aa38 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -10,20 +10,29 @@ import { appContextService } from '../..'; import { handleStateMachine } from './integrations_state_machine'; -const getTestDefinition = (mockEvent1: any, mockEvent2: any, mockEvent3: any, context?: any) => { +const getTestDefinition = ( + mockEvent1: any, + mockEvent2: any, + mockEvent3: any, + context?: any, + onPostTransition?: any +) => { return { context, states: { state1: { onTransitionTo: mockEvent1, + onPostTransition, nextState: 'state2', }, state2: { onTransitionTo: mockEvent2, + onPostTransition, nextState: 'state3', }, state3: { onTransitionTo: mockEvent3, + onPostTransition, nextState: 'end', }, }, @@ -118,4 +127,29 @@ describe('handleStateMachine', () => { 'state: state3 - status success - stateResult: undefined - nextState: end' ); }); + + it('should execute postTransition function after the transition is complete', async () => { + const mockEventState1 = jest.fn(); + const mockEventState2 = jest.fn(); + const mockEventState3 = jest.fn(); + const mockPostTransition = jest.fn(); + const testDefinition = getTestDefinition( + mockEventState1, + mockEventState2, + mockEventState3, + undefined, + mockPostTransition + ); + await handleStateMachine('state1', testDefinition); + + expect(mockEventState1).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockEventState2).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockEventState3).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executing post transition function: mockConstructor' + ); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index b79b1f83ce937..68ceea916d590 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -9,6 +9,7 @@ export interface State { onTransitionTo: any; nextState?: string; currentStatus?: string; + onPostTransition?: any; } export type StatusName = 'success' | 'failed' | 'pending'; @@ -45,10 +46,17 @@ async function handleState(currentStateName: string, definition: StateMachineDef } else { currentStatus = 'failed'; } - // update SO with current state data logger.debug( `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` ); + if (typeof currentState.onPostTransition === 'function') { + try { + await currentState.onPostTransition.call(undefined, context); + logger.debug(`Executing post transition function: ${currentState.onPostTransition.name}`); + } catch (error) { + logger.warn(`Error during execution of post transition function: ${error.message}`); + } + } if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { handleState(currentState.nextState, definition); } else { From 28d4de068bc896a758651933d0191899bff7190a Mon Sep 17 00:00:00 2001 From: criamico Date: Tue, 19 Mar 2024 12:43:00 +0100 Subject: [PATCH 06/38] Save data between iterations --- .../integrations_state_machine.test.ts | 31 ++++++++++++------- .../packages/integrations_state_machine.ts | 18 +++++++---- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index 574c849e8aa38..882fa93ae3f9b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -62,13 +62,13 @@ describe('handleStateMachine', () => { expect(mockEventState2).toHaveBeenCalledTimes(1); expect(mockEventState3).toHaveBeenCalledTimes(1); expect(mockContract.logger?.debug).toHaveBeenCalledWith( - 'state: state1 - status success - stateResult: undefined - nextState: state2' + 'Executed state: state1 with status: success - nextState: state2' ); expect(mockContract.logger?.debug).toHaveBeenCalledWith( - 'state: state2 - status success - stateResult: undefined - nextState: state3' + 'Executed state: state2 with status: success - nextState: state3' ); expect(mockContract.logger?.debug).toHaveBeenCalledWith( - 'state: state3 - status success - stateResult: undefined - nextState: end' + 'Executed state: state3 with status: success - nextState: end' ); }); @@ -100,9 +100,11 @@ describe('handleStateMachine', () => { ); }); - it('should call the onTransition function with the provided data', async () => { - const mockEventState1 = jest.fn(); - const mockEventState2 = jest.fn(); + it('should call the onTransition function with context data and the return value is saved for the next iteration', async () => { + const mockEventState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); + const mockEventState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ promiseData: {} })); const mockEventState3 = jest.fn(); const contextData = { testData: 'test' }; const testDefinition = getTestDefinition( @@ -114,17 +116,24 @@ describe('handleStateMachine', () => { await handleStateMachine('state1', testDefinition); expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockEventState2).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockEventState3).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockEventState2).toHaveBeenCalledWith({ + testData: 'test', + arrayData: ['test1', 'test2'], + }); + expect(mockEventState3).toHaveBeenCalledWith({ + testData: 'test', + arrayData: ['test1', 'test2'], + promiseData: {}, + }); expect(mockContract.logger?.debug).toHaveBeenCalledWith( - 'state: state1 - status success - stateResult: undefined - nextState: state2' + 'Executed state: state1 with status: success - nextState: state2' ); expect(mockContract.logger?.debug).toHaveBeenCalledWith( - 'state: state2 - status success - stateResult: undefined - nextState: state3' + 'Executed state: state2 with status: success - nextState: state3' ); expect(mockContract.logger?.debug).toHaveBeenCalledWith( - 'state: state3 - status success - stateResult: undefined - nextState: end' + 'Executed state: state3 with status: success - nextState: end' ); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 68ceea916d590..5d243c9ea03c7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -23,19 +23,25 @@ export async function handleStateMachine( startState: string, definition: StateMachineDefinition ) { - await handleState(startState, definition); + await handleState(startState, definition, definition.context); } -async function handleState(currentStateName: string, definition: StateMachineDefinition) { +async function handleState( + currentStateName: string, + definition: StateMachineDefinition, + context: any +) { const logger = appContextService.getLogger(); - const { states, context } = definition; + const { states } = definition; const currentState = states[currentStateName]; let currentStatus = 'pending'; let stateResult; + let updatedContext; if (typeof currentState.onTransitionTo === 'function') { try { stateResult = await currentState.onTransitionTo.call(undefined, context); + updatedContext = { ...context, ...stateResult }; currentStatus = 'success'; } catch (error) { currentStatus = 'failed'; @@ -47,18 +53,18 @@ async function handleState(currentStateName: string, definition: StateMachineDef currentStatus = 'failed'; } logger.debug( - `state: ${currentStateName} - status ${currentStatus} - stateResult: ${stateResult} - nextState: ${currentState.nextState}` + `Executed state: ${currentStateName} with status: ${currentStatus} - nextState: ${currentState.nextState}` ); if (typeof currentState.onPostTransition === 'function') { try { - await currentState.onPostTransition.call(undefined, context); + await currentState.onPostTransition.call(undefined, updatedContext); logger.debug(`Executing post transition function: ${currentState.onPostTransition.name}`); } catch (error) { logger.warn(`Error during execution of post transition function: ${error.message}`); } } if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { - handleState(currentState.nextState, definition); + handleState(currentState.nextState, definition, updatedContext); } else { return; } From 59cd76767bddf5b1fffad40050b35192b23afc00 Mon Sep 17 00:00:00 2001 From: criamico Date: Wed, 20 Mar 2024 12:28:09 +0100 Subject: [PATCH 07/38] Improvements to state machine --- .../integrations_state_machine.test.ts | 87 ++++++++++++++++++- .../packages/integrations_state_machine.ts | 15 ++-- 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index 882fa93ae3f9b..b6ad53ffa8358 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -105,7 +105,7 @@ describe('handleStateMachine', () => { const mockEventState2 = jest .fn() .mockImplementation(() => Promise.resolve({ promiseData: {} })); - const mockEventState3 = jest.fn(); + const mockEventState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); const contextData = { testData: 'test' }; const testDefinition = getTestDefinition( mockEventState1, @@ -137,6 +137,91 @@ describe('handleStateMachine', () => { ); }); + it('should save the return data from transitions also when return type is function', async () => { + const mockEventState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); + const state2Result = () => { + return { + innerData: 'test', + }; + }; + const mockEventState2 = jest.fn().mockImplementation(() => { + return state2Result; + }); + const mockEventState3 = jest.fn(); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockEventState1, + mockEventState2, + mockEventState3, + contextData + ); + + await handleStateMachine('state1', testDefinition); + expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockEventState2).toHaveBeenCalledWith({ + testData: 'test', + arrayData: ['test1', 'test2'], + state2Result, + }); + expect(mockEventState3).toHaveBeenCalledWith({ + testData: 'test', + arrayData: ['test1', 'test2'], + state2Result, + }); + + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state1 with status: success - nextState: state2' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state2 with status: success - nextState: state3' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state3 with status: success - nextState: end' + ); + }); + + it.skip('should return updated context data', async () => { + const mockEventState1 = jest + .fn() + .mockImplementation(() => Promise.resolve({ promiseData: {} })); + const state2Result = () => { + return { + innerData: 'test', + }; + }; + const mockEventState2 = jest.fn().mockImplementation(() => { + return state2Result; + }); + const mockEventState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockEventState1, + mockEventState2, + mockEventState3, + contextData + ); + + const updatedContext = await handleStateMachine('state1', testDefinition); + expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockEventState2).toHaveBeenCalledWith({ + testData: 'test', + promiseData: {}, + state2Result, + }); + expect(mockEventState3).toHaveBeenCalledWith({ + testData: 'test', + promiseData: {}, + state2Result, + }); + + expect(updatedContext).toEqual({ + testData: 'test', + promiseData: {}, + state2Result, + lastData: ['test3'], + }); + }); + it('should execute postTransition function after the transition is complete', async () => { const mockEventState1 = jest.fn(); const mockEventState2 = jest.fn(); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 5d243c9ea03c7..852b60e42a79b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -36,12 +36,17 @@ async function handleState( const currentState = states[currentStateName]; let currentStatus = 'pending'; let stateResult; - let updatedContext; - + let updatedContext = { ...context }; if (typeof currentState.onTransitionTo === 'function') { try { - stateResult = await currentState.onTransitionTo.call(undefined, context); - updatedContext = { ...context, ...stateResult }; + stateResult = await currentState.onTransitionTo.call(undefined, updatedContext); + // check if is a function/promise + if (typeof stateResult === 'function') { + const promiseName = `${currentStateName}Result`; + updatedContext[promiseName] = stateResult; + } else { + updatedContext = { ...updatedContext, ...stateResult }; + } currentStatus = 'success'; } catch (error) { currentStatus = 'failed'; @@ -64,7 +69,7 @@ async function handleState( } } if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { - handleState(currentState.nextState, definition, updatedContext); + await handleState(currentState.nextState, definition, updatedContext); } else { return; } From fffba51b5311836aa781128615be8d5e2ee72251 Mon Sep 17 00:00:00 2001 From: criamico Date: Wed, 20 Mar 2024 15:41:10 +0100 Subject: [PATCH 08/38] Remove wrapper function and return updatedContext --- .../integrations_state_machine.test.ts | 20 +++++++++---------- .../packages/integrations_state_machine.ts | 13 +++--------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index b6ad53ffa8358..a05ca81ca27ea 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -8,7 +8,7 @@ import { createAppContextStartContractMock } from '../../../mocks'; import { appContextService } from '../..'; -import { handleStateMachine } from './integrations_state_machine'; +import { handleState } from './integrations_state_machine'; const getTestDefinition = ( mockEvent1: any, @@ -39,7 +39,7 @@ const getTestDefinition = ( }; }; -describe('handleStateMachine', () => { +describe('handleState', () => { let mockContract: ReturnType; beforeEach(async () => { // prevents `Logger not set.` and other appContext errors @@ -57,7 +57,7 @@ describe('handleStateMachine', () => { const mockEventState3 = jest.fn(); const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); - await handleStateMachine('state1', testDefinition); + await handleState('state1', testDefinition, testDefinition.context); expect(mockEventState1).toHaveBeenCalledTimes(1); expect(mockEventState2).toHaveBeenCalledTimes(1); expect(mockEventState3).toHaveBeenCalledTimes(1); @@ -77,7 +77,7 @@ describe('handleStateMachine', () => { const mockEventState2 = jest.fn(); const mockEventState3 = jest.fn(); const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); - await handleStateMachine('state2', testDefinition); + await handleState('state2', testDefinition, testDefinition.context); expect(mockEventState1).toHaveBeenCalledTimes(0); expect(mockEventState2).toHaveBeenCalledTimes(1); @@ -90,7 +90,7 @@ describe('handleStateMachine', () => { const mockEventState2 = jest.fn(); const mockEventState3 = jest.fn(); const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); - await handleStateMachine('state1', testDefinition); + await handleState('state1', testDefinition, testDefinition.context); expect(mockEventState1).toHaveBeenCalledTimes(1); expect(mockEventState2).toHaveBeenCalledTimes(0); @@ -114,7 +114,7 @@ describe('handleStateMachine', () => { contextData ); - await handleStateMachine('state1', testDefinition); + await handleState('state1', testDefinition, testDefinition.context); expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); expect(mockEventState2).toHaveBeenCalledWith({ testData: 'test', @@ -156,7 +156,7 @@ describe('handleStateMachine', () => { contextData ); - await handleStateMachine('state1', testDefinition); + await handleState('state1', testDefinition, testDefinition.context); expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); expect(mockEventState2).toHaveBeenCalledWith({ testData: 'test', @@ -180,7 +180,7 @@ describe('handleStateMachine', () => { ); }); - it.skip('should return updated context data', async () => { + it('should return updated context data', async () => { const mockEventState1 = jest .fn() .mockImplementation(() => Promise.resolve({ promiseData: {} })); @@ -201,7 +201,7 @@ describe('handleStateMachine', () => { contextData ); - const updatedContext = await handleStateMachine('state1', testDefinition); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); expect(mockEventState2).toHaveBeenCalledWith({ testData: 'test', @@ -234,7 +234,7 @@ describe('handleStateMachine', () => { undefined, mockPostTransition ); - await handleStateMachine('state1', testDefinition); + await handleState('state1', testDefinition, testDefinition.context); expect(mockEventState1).toHaveBeenCalled(); expect(mockPostTransition).toHaveBeenCalled(); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 852b60e42a79b..0fe3455599991 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -19,14 +19,7 @@ export interface StateMachineDefinition { states: StateMachineStates; } -export async function handleStateMachine( - startState: string, - definition: StateMachineDefinition -) { - await handleState(startState, definition, definition.context); -} - -async function handleState( +export async function handleState( currentStateName: string, definition: StateMachineDefinition, context: any @@ -69,8 +62,8 @@ async function handleState( } } if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { - await handleState(currentState.nextState, definition, updatedContext); + return await handleState(currentState.nextState, definition, updatedContext); } else { - return; + return updatedContext; } } From 6267ec6d5f2daf01f8f9b12fe5c3e05a02b8815e Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 21 Mar 2024 16:20:59 +0100 Subject: [PATCH 09/38] Implement step functions --- .../services/epm/packages/_install_package.ts | 83 +++- .../_state_machine_package_install.ts | 98 ++--- .../server/services/epm/packages/install.ts | 77 ---- .../services/epm/packages/install_steps.ts | 414 ++++++++++++++++++ .../packages/integrations_state_machine.ts | 4 +- 5 files changed, 525 insertions(+), 151 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index f68cef6ccd31f..242671435b522 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -21,6 +21,7 @@ import type { PackageInstallContext } from '../../../../common/types'; import { getNormalizedDataStreams } from '../../../../common/services'; import { + MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT, @@ -44,12 +45,12 @@ import { installTransforms } from '../elasticsearch/transform/install'; import { installMlModel } from '../elasticsearch/ml_model'; import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; import { saveArchiveEntriesFromAssetsMap } from '../archive/storage'; -import { PackageSavedObjectConflictError } from '../../../errors'; +import { ConcurrentInstallOperationError, PackageSavedObjectConflictError } from '../../../errors'; import { appContextService, packagePolicyService } from '../..'; import { auditLoggingService } from '../../audit_logging'; -import { createRestartInstallation } from './install'; +import { createInstallation, restartInstallation } from './install'; import { withPackageSpan } from './utils'; import { clearLatestFailedAttempts } from './install_errors_helpers'; import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; @@ -96,18 +97,60 @@ export async function _installPackage({ const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo; try { - await createRestartInstallation({ - savedObjectsClient, - logger, - packageInfo, - installSource, - spaceId, - force, - verificationResult, - installedPkg, - }); - logger.debug(`Package install - Installing Kibana assets`); - // step install_kibana_assets + if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); + + // if the installation is currently running, don't try to install + // instead, only return already installed assets + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + logger.debug(`Package install - Installation is running and has exceeded timeout`); + + if (force) { + logger.debug(`Package install - Forced installation, restarting`); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } + } else { + // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL + // (it might be stuck) update the saved object and proceed + logger.debug( + `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` + ); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } + } else { + logger.debug(`Package install - Create installation`); + await createInstallation({ + savedObjectsClient, + packageInfo, + installSource, + spaceId, + verificationResult, + }); + } + const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => installKibanaAssetsAndReferences({ savedObjectsClient, @@ -153,8 +196,7 @@ export async function _installPackage({ ) )); } - // step install_ml_model - // installs ml models + logger.debug(`Package install - installing ML models`); esReferences = await withPackageSpan('Install ML models', () => installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) @@ -162,8 +204,6 @@ export async function _installPackage({ let indexTemplates: IndexTemplateEntry[] = []; - // step install_index_template_pipelines - // it should contain the case for integration and the one for inputs if (packageInfo.type === 'integration') { logger.debug( `Package install - Installing index templates and pipelines, packageInfo.type ${packageInfo.type}` @@ -213,7 +253,7 @@ export async function _installPackage({ indexTemplates = installedTemplates; } } - // step remove_legacy_templates + try { logger.debug(`Package install - Removing legacy templates`); await removeLegacyTemplates({ packageInfo, esClient, logger }); @@ -221,7 +261,6 @@ export async function _installPackage({ logger.warn(`Error removing legacy templates: ${e.message}`); } - // step update_current_write_indices // update current backing indices of each data stream logger.debug(`Package install - Updating backing indices of each data stream`); await withPackageSpan('Update write indices', () => @@ -231,7 +270,7 @@ export async function _installPackage({ }) ); logger.debug(`Package install - Installing transforms`); - // steps install_transforms + ({ esReferences } = await withPackageSpan('Install transforms', () => installTransforms({ packageInstallContext, @@ -247,7 +286,6 @@ export async function _installPackage({ // If this is an update or retrying an update, delete the previous version's pipelines // Top-level pipeline assets will not be removed on upgrade as of ml model package addition which requires previous // assets to remain installed. This is a temporary solution - more robust solution tracked here https://github.com/elastic/kibana/issues/115035 - // steps delete_previous_pipelines - should contain the two ifs if ( paths.filter((path) => isTopLevelPipeline(path)).length === 0 && (installType === 'update' || installType === 'reupdate') && @@ -283,7 +321,6 @@ export async function _installPackage({ } const installedKibanaAssetsRefs = await kibanaAssetPromise; - // step save_archive_entries_from_assets_map logger.debug(`Package install - Updating archive entries`); const packageAssetResults = await withPackageSpan('Update archive entries', () => saveArchiveEntriesFromAssetsMap({ diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts index fe90cef2bdba2..7c1e4b6485e0c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts @@ -16,21 +16,38 @@ import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging import type { HTTPAuthorizationHeader } from '../../../../common/http_authorization_header'; import type { PackageInstallContext } from '../../../../common/types'; +import type { PackageAssetReference } from '../../../types'; import type { Installation, InstallType, InstallSource, PackageVerificationResult, + EsAssetReference, + KibanaAssetReference, + IndexTemplateEntry, } from '../../../types'; -import { createRestartInstallation } from './install'; +import { + stepCreateRestartInstallation, + stepInstallKibanaAssets, + stepInstallILMPolicies, + stepInstallMlModel, + stepInstallIndexTemplatePipelines, + stepRemoveLegacyTemplates, + stepUpdateCurrentWriteIndices, + stepInstallTransforms, + stepDeletePreviousPipelines, + stepSaveArchiveEntries, + stepSaveSystemObject, +} from './install_steps'; import type { StateMachineDefinition } from './integrations_state_machine'; -import { handleStateMachine } from './integrations_state_machine'; +import { handleState } from './integrations_state_machine'; export const installStateNames = [ 'create_restart_installation', 'install_kibana_assets', + 'install_ilm_policies', 'install_ml_model', 'install_index_template_pipelines', 'remove_legacy_templates', @@ -44,24 +61,7 @@ export const installStateNames = [ type StateNamesTuple = typeof installStateNames; type StateNames = StateNamesTuple[number]; -export async function _stateMachineInstallPackage({ - savedObjectsClient, - savedObjectsImporter, - savedObjectTagAssignmentService, - savedObjectTagClient, - esClient, - logger, - installedPkg, - packageInstallContext, - installType, - installSource, - spaceId, - force, - verificationResult, - authorizationHeader, - ignoreMappingUpdateErrors, - skipDataStreamRollover, -}: { +export interface InstallContext { savedObjectsClient: SavedObjectsClientContract; savedObjectsImporter: Pick; savedObjectTagAssignmentService: IAssignmentService; @@ -78,68 +78,68 @@ export async function _stateMachineInstallPackage({ authorizationHeader?: HTTPAuthorizationHeader | null; ignoreMappingUpdateErrors?: boolean; skipDataStreamRollover?: boolean; -}) { - const { packageInfo, paths } = packageInstallContext; - const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo; - // our install states + indexTemplates: IndexTemplateEntry[]; + packageAssetRefs: PackageAssetReference[]; + // output values + esReferences: EsAssetReference[]; + kibanaAssetPromise: Promise; +} + +export async function _stateMachineInstallPackage({ context }: { context: InstallContext }) { const installStates: StateMachineDefinition = { - context: { - savedObjectsClient, - savedObjectsImporter, - savedObjectTagAssignmentService, - savedObjectTagClient, - pkgName, - pkgTitle, - packageInstallContext, - paths, - installedPkg, - logger, - spaceId, - assetTags: packageInfo?.asset_tags, - }, + context, states: { create_restart_installation: { nextState: 'install_kibana_assets', - onTransitionTo: createRestartInstallation, + onTransitionTo: stepCreateRestartInstallation, }, install_kibana_assets: { - onTransitionTo: () => undefined, + onTransitionTo: stepInstallKibanaAssets, + nextState: 'install_ilm_policies', + }, + install_ilm_policies: { + onTransitionTo: stepInstallILMPolicies, nextState: 'install_ml_model', }, install_ml_model: { - onTransitionTo: () => undefined, + onTransitionTo: stepInstallMlModel, nextState: 'install_index_template_pipelines', }, install_index_template_pipelines: { - onTransitionTo: () => undefined, + onTransitionTo: stepInstallIndexTemplatePipelines, nextState: 'remove_legacy_templates', }, remove_legacy_templates: { - onTransitionTo: () => undefined, + onTransitionTo: stepRemoveLegacyTemplates, nextState: 'update_current_write_indices', }, update_current_write_indices: { - onTransitionTo: () => undefined, + onTransitionTo: stepUpdateCurrentWriteIndices, nextState: 'install_transforms', }, install_transforms: { - onTransitionTo: () => undefined, + onTransitionTo: stepInstallTransforms, nextState: 'delete_previous_pipelines', }, delete_previous_pipelines: { - onTransitionTo: () => undefined, + onTransitionTo: stepDeletePreviousPipelines, nextState: 'save_archive_entries_from_assets_map', }, save_archive_entries_from_assets_map: { - onTransitionTo: () => undefined, + onTransitionTo: stepSaveArchiveEntries, nextState: 'update_so', }, update_so: { - onTransitionTo: () => undefined, + onTransitionTo: stepSaveSystemObject, nextState: 'end', }, }, }; - await handleStateMachine('create_restart_installation', installStates); + const { installedKibanaAssetsRefs, esReferences } = await handleState( + 'create_restart_installation', + installStates, + installStates.context + ); + return [...installedKibanaAssetsRefs, ...esReferences]; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index bcef62620492b..43b0c9d68a04c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -971,83 +971,6 @@ export const updateInstallStatusToFailed = async ({ } }; -export async function createRestartInstallation({ - savedObjectsClient, - logger, - packageInfo, - installSource, - spaceId, - force, - verificationResult, - installedPkg, -}: { - savedObjectsClient: SavedObjectsClientContract; - logger: Logger; - packageInfo: InstallablePackage; - installSource: InstallSource; - spaceId: string; - force?: boolean; - verificationResult?: PackageVerificationResult; - installedPkg?: SavedObject; -}) { - const { name: pkgName, version: pkgVersion } = packageInfo; - // if some installation already exists - if (installedPkg) { - const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; - const hasExceededTimeout = - Date.now() - Date.parse(installedPkg.attributes.install_started_at) < - MAX_TIME_COMPLETE_INSTALL; - logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); - - // if the installation is currently running, don't try to install - // instead, only return already installed assets - if (isStatusInstalling && hasExceededTimeout) { - // If this is a forced installation, ignore the timeout and restart the installation anyway - logger.debug(`Package install - Installation is running and has exceeded timeout`); - - if (force) { - logger.debug(`Package install - Forced installation, restarting`); - await restartInstallation({ - savedObjectsClient, - pkgName, - pkgVersion, - installSource, - verificationResult, - }); - } else { - throw new ConcurrentInstallOperationError( - `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ - pkgVersion || 'unknown' - } detected, aborting.` - ); - } - } else { - // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL - // (it might be stuck) update the saved object and proceed - logger.debug( - `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` - ); - await restartInstallation({ - savedObjectsClient, - pkgName, - pkgVersion, - installSource, - verificationResult, - }); - } - } else { - logger.debug(`Package install - Create installation`); - // step create_installation - await createInstallation({ - savedObjectsClient, - packageInfo, - installSource, - spaceId, - verificationResult, - }); - } -} - export async function restartInstallation(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts new file mode 100644 index 0000000000000..d37f74abdb3c4 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts @@ -0,0 +1,414 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ConcurrentInstallOperationError } from '../../../errors'; +import { + MAX_TIME_COMPLETE_INSTALL, + ASSETS_SAVED_OBJECT_TYPE, + PACKAGES_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, +} from '../../../constants'; +import type { PackageAssetReference, Installation } from '../../../types'; + +import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; + +import { appContextService, packagePolicyService } from '../..'; + +import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; +import { installILMPolicy } from '../elasticsearch/ilm/install'; +import { installMlModel } from '../elasticsearch/ml_model'; + +import { getNormalizedDataStreams } from '../../../../common/services'; + +import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy'; + +import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; + +import { installTransforms } from '../elasticsearch/transform/install'; + +import { isTopLevelPipeline, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline'; + +import { auditLoggingService } from '../../audit_logging'; + +import { saveArchiveEntriesFromAssetsMap } from '../archive/storage'; + +import { restartInstallation, createInstallation } from './install'; +import { withPackageSpan } from './utils'; +import type { InstallContext } from './_state_machine_package_install'; +import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; + +export async function stepCreateRestartInstallation({ context }: { context: InstallContext }) { + const { + savedObjectsClient, + logger, + installSource, + packageInstallContext, + spaceId, + force, + verificationResult, + installedPkg, + } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName, version: pkgVersion } = packageInfo; + // if some installation already exists + if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); + + // if the installation is currently running, don't try to install + // instead, only return already installed assets + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + logger.debug(`Package install - Installation is running and has exceeded timeout`); + + if (force) { + logger.debug(`Package install - Forced installation, restarting`); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } + } else { + // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL + // (it might be stuck) update the saved object and proceed + logger.debug( + `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` + ); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } + } else { + logger.debug(`Package install - Create installation`); + // step create_installation + await createInstallation({ + savedObjectsClient, + packageInfo, + installSource, + spaceId, + verificationResult, + }); + } + + // Use a shared array that is updated by each operation. This allows each operation to accurately update the + // installation object with it's references without requiring a refresh of the SO index on each update (faster). + const esReferences = installedPkg?.attributes.installed_es ?? []; + return { esReferences }; +} + +export async function stepInstallKibanaAssets({ context }: { context: InstallContext }) { + const { + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + logger, + installedPkg, + packageInstallContext, + spaceId, + } = context; + const { packageInfo, paths } = packageInstallContext; + const { name: pkgName, title: pkgTitle } = packageInfo; + + logger.debug(`Package install - Installing Kibana assets`); + // step install_kibana_assets + const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => + installKibanaAssetsAndReferences({ + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + pkgName, + pkgTitle, + packageInstallContext, + paths, + installedPkg, + logger, + spaceId, + assetTags: packageInfo?.asset_tags, + }) + ); + return { kibanaAssetPromise }; +} + +export async function stepInstallILMPolicies({ context }: { context: InstallContext }) { + const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; + let updatedEsReferences; + // currently only the base package has an ILM policy + // at some point ILM policies can be installed/modified + // per data stream and we should then save them + const isILMPoliciesDisabled = + appContextService.getConfig()?.internal?.disableILMPolicies ?? false; + if (!isILMPoliciesDisabled) { + updatedEsReferences = await withPackageSpan('Install ILM policies', () => + installILMPolicy(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) + ); + logger.debug(`Package install - Installing Data Stream ILM policies`); + ({ esReferences: updatedEsReferences } = await withPackageSpan( + 'Install Data Stream ILM policies', + () => + installIlmForDataStream( + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences + ) + )); + } + return { esReferences: updatedEsReferences }; +} + +export async function stepInstallMlModel({ context }: { context: InstallContext }) { + const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; + + // installs ml models + logger.debug(`Package install - installing ML models`); + const updatedEsReferences = await withPackageSpan('Install ML models', () => + installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) + ); + return { esReferences: updatedEsReferences }; +} + +export async function stepInstallIndexTemplatePipelines({ context }: { context: InstallContext }) { + const { + esClient, + savedObjectsClient, + packageInstallContext, + logger, + installedPkg, + esReferences, + } = context; + const { packageInfo } = packageInstallContext; + + if (packageInfo.type === 'integration') { + logger.debug( + `Package install - Installing index templates and pipelines, packageInfo.type ${packageInfo.type}` + ); + const { installedTemplates, esReferences: templateEsReferences } = + await installIndexTemplatesAndPipelines({ + installedPkg: installedPkg ? installedPkg.attributes : undefined, + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + }); + return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; + } + + if (packageInfo.type === 'input' && installedPkg) { + // input packages create their data streams during package policy creation + // we must use installed_es to infer which streams exist first then + // we can install the new index templates + logger.debug(`Package install - packageInfo.type ${packageInfo.type}`); + const dataStreamNames = installedPkg.attributes.installed_es + .filter((ref) => ref.type === 'index_template') + // index templates are named {type}-{dataset}, remove everything before first hyphen + .map((ref) => ref.id.replace(/^[^-]+-/, '')); + + const dataStreams = dataStreamNames.flatMap((dataStreamName) => + getNormalizedDataStreams(packageInfo, dataStreamName) + ); + + if (dataStreams.length) { + logger.debug( + `Package install - installing index templates and pipelines with datastreams length ${dataStreams.length}` + ); + const { installedTemplates, esReferences: templateEsReferences } = + await installIndexTemplatesAndPipelines({ + installedPkg: installedPkg ? installedPkg.attributes : undefined, + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + onlyForDataStreams: dataStreams, + }); + return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; + } + } +} + +export async function stepRemoveLegacyTemplates({ context }: { context: InstallContext }) { + const { esClient, packageInstallContext, logger } = context; + const { packageInfo } = packageInstallContext; + try { + logger.debug(`Package install - Removing legacy templates`); + await removeLegacyTemplates({ packageInfo, esClient, logger }); + } catch (e) { + logger.warn(`Error removing legacy templates: ${e.message}`); + } +} + +export async function stepUpdateCurrentWriteIndices({ context }: { context: InstallContext }) { + const { esClient, logger, ignoreMappingUpdateErrors, skipDataStreamRollover, indexTemplates } = + context; + + // update current backing indices of each data stream + logger.debug(`Package install - Updating backing indices of each data stream`); + await withPackageSpan('Update write indices', () => + updateCurrentWriteIndices(esClient, logger, indexTemplates, { + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }) + ); +} + +export async function stepInstallTransforms({ context }: { context: InstallContext }) { + const { + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + force, + authorizationHeader, + } = context; + logger.debug(`Package install - Installing transforms`); + // steps install_transforms + const res = await withPackageSpan('Install transforms', () => + installTransforms({ + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + force, + authorizationHeader, + }) + ); + + return { esReferences: res.esReferences }; +} + +export async function stepDeletePreviousPipelines({ context }: { context: InstallContext }) { + const { + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + installType, + installedPkg, + } = context; + const { packageInfo, paths } = packageInstallContext; + const { name: pkgName } = packageInfo; + let updatedESReferences; + // If this is an update or retrying an update, delete the previous version's pipelines + // Top-level pipeline assets will not be removed on upgrade as of ml model package addition which requires previous + // assets to remain installed. This is a temporary solution - more robust solution tracked here https://github.com/elastic/kibana/issues/115035 + if ( + paths.filter((path) => isTopLevelPipeline(path)).length === 0 && + (installType === 'update' || installType === 'reupdate') && + installedPkg + ) { + logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); + updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => + deletePreviousPipelines( + esClient, + savedObjectsClient, + pkgName, + installedPkg!.attributes.version, + esReferences + ) + ); + } + // pipelines from a different version may have installed during a failed update + if (installType === 'rollback' && installedPkg) { + logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); + updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => + deletePreviousPipelines( + esClient, + savedObjectsClient, + pkgName, + installedPkg!.attributes.install_version, + esReferences + ) + ); + } + return { esReferences: updatedESReferences }; +} + +export async function stepSaveArchiveEntries({ context }: { context: InstallContext }) { + const { packageInstallContext, savedObjectsClient, logger, installSource, kibanaAssetPromise } = + context; + const installedKibanaAssetsRefs = await kibanaAssetPromise; + + const { packageInfo } = packageInstallContext; + logger.debug(`Package install - Updating archive entries`); + const packageAssetResults = await withPackageSpan('Update archive entries', () => + saveArchiveEntriesFromAssetsMap({ + savedObjectsClient, + assetsMap: packageInstallContext.assetsMap, + paths: packageInstallContext.paths, + packageInfo, + installSource, + }) + ); + const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map( + (result) => ({ + id: result.id, + type: ASSETS_SAVED_OBJECT_TYPE, + }) + ); + + return { packageAssetRefs, installedKibanaAssetsRefs }; +} + +export async function stepSaveSystemObject({ context }: { context: InstallContext }) { + const { packageInstallContext, savedObjectsClient, logger, esClient } = context; + + const { packageInfo } = packageInstallContext; + const { name: pkgName } = packageInfo; + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + + // Need to refetch the installation again to retrieve all the attributes + const updatedPackage = await savedObjectsClient.get( + PACKAGES_SAVED_OBJECT_TYPE, + pkgName + ); + logger.debug(`Package install - Install status ${updatedPackage?.attributes?.install_status}`); + // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its + // associated package policies after installation + if (updatedPackage.attributes.keep_policies_up_to_date) { + await withPackageSpan('Upgrade package policies', async () => { + const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, { + page: 1, + perPage: SO_SEARCH_LIMIT, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`, + }); + logger.debug( + `Package install - Package is flagged with keep_policies_up_to_date, upgrading its associated package policies ${policyIdsToUpgrade}` + ); + await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); + }); + } + logger.debug(`Package install - Installation complete`); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 0fe3455599991..9c9300daeb8cb 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -22,8 +22,8 @@ export interface StateMachineDefinition { export async function handleState( currentStateName: string, definition: StateMachineDefinition, - context: any -) { + context: { [key: string]: any } +): Promise<{ [key: string]: any }> { const logger = appContextService.getLogger(); const { states } = definition; const currentState = states[currentStateName]; From 8c00a9bc790e9214b95733f8b72b3c85074bafa8 Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 22 Mar 2024 10:21:28 +0100 Subject: [PATCH 10/38] remove changes to _install_packages --- .../fleet/server/services/epm/packages/_install_package.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 242671435b522..857ff668a6e1c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -151,6 +151,7 @@ export async function _installPackage({ }); } + logger.debug(`Package install - Installing Kibana assets`); const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => installKibanaAssetsAndReferences({ savedObjectsClient, @@ -197,6 +198,7 @@ export async function _installPackage({ )); } + // installs ml models logger.debug(`Package install - installing ML models`); esReferences = await withPackageSpan('Install ML models', () => installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) @@ -269,6 +271,7 @@ export async function _installPackage({ skipDataStreamRollover, }) ); + logger.debug(`Package install - Installing transforms`); ({ esReferences } = await withPackageSpan('Install transforms', () => @@ -343,7 +346,7 @@ export async function _installPackage({ id: pkgName, savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, }); - // step update_SO + logger.debug(`Package install - Updating install status`); await withPackageSpan('Update install status', () => savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { From 04f0e2d7eeb447043e1f596781d863db2011d92a Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 22 Mar 2024 10:39:37 +0100 Subject: [PATCH 11/38] Naming changes and tests --- .../services/epm/packages/_install_package.ts | 5 +- .../_state_machine_package_install.ts | 22 +-- .../integrations_state_machine.test.ts | 165 +++++++++++------- .../packages/integrations_state_machine.ts | 6 +- 4 files changed, 117 insertions(+), 81 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 857ff668a6e1c..33fa489c81466 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -98,6 +98,7 @@ export async function _installPackage({ try { if (installedPkg) { + // if some installation already exists const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; const hasExceededTimeout = Date.now() - Date.parse(installedPkg.attributes.install_started_at) < @@ -150,7 +151,6 @@ export async function _installPackage({ verificationResult, }); } - logger.debug(`Package install - Installing Kibana assets`); const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => installKibanaAssetsAndReferences({ @@ -271,9 +271,7 @@ export async function _installPackage({ skipDataStreamRollover, }) ); - logger.debug(`Package install - Installing transforms`); - ({ esReferences } = await withPackageSpan('Install transforms', () => installTransforms({ packageInstallContext, @@ -346,7 +344,6 @@ export async function _installPackage({ id: pkgName, savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, }); - logger.debug(`Package install - Updating install status`); await withPackageSpan('Update install status', () => savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts index 7c1e4b6485e0c..59c5308996bcb 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts @@ -92,46 +92,46 @@ export async function _stateMachineInstallPackage({ context }: { context: Instal states: { create_restart_installation: { nextState: 'install_kibana_assets', - onTransitionTo: stepCreateRestartInstallation, + onTransition: stepCreateRestartInstallation, }, install_kibana_assets: { - onTransitionTo: stepInstallKibanaAssets, + onTransition: stepInstallKibanaAssets, nextState: 'install_ilm_policies', }, install_ilm_policies: { - onTransitionTo: stepInstallILMPolicies, + onTransition: stepInstallILMPolicies, nextState: 'install_ml_model', }, install_ml_model: { - onTransitionTo: stepInstallMlModel, + onTransition: stepInstallMlModel, nextState: 'install_index_template_pipelines', }, install_index_template_pipelines: { - onTransitionTo: stepInstallIndexTemplatePipelines, + onTransition: stepInstallIndexTemplatePipelines, nextState: 'remove_legacy_templates', }, remove_legacy_templates: { - onTransitionTo: stepRemoveLegacyTemplates, + onTransition: stepRemoveLegacyTemplates, nextState: 'update_current_write_indices', }, update_current_write_indices: { - onTransitionTo: stepUpdateCurrentWriteIndices, + onTransition: stepUpdateCurrentWriteIndices, nextState: 'install_transforms', }, install_transforms: { - onTransitionTo: stepInstallTransforms, + onTransition: stepInstallTransforms, nextState: 'delete_previous_pipelines', }, delete_previous_pipelines: { - onTransitionTo: stepDeletePreviousPipelines, + onTransition: stepDeletePreviousPipelines, nextState: 'save_archive_entries_from_assets_map', }, save_archive_entries_from_assets_map: { - onTransitionTo: stepSaveArchiveEntries, + onTransition: stepSaveArchiveEntries, nextState: 'update_so', }, update_so: { - onTransitionTo: stepSaveSystemObject, + onTransition: stepSaveSystemObject, nextState: 'end', }, }, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index a05ca81ca27ea..40044ed53d674 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -11,9 +11,9 @@ import { appContextService } from '../..'; import { handleState } from './integrations_state_machine'; const getTestDefinition = ( - mockEvent1: any, - mockEvent2: any, - mockEvent3: any, + mockOnTransition1: any, + mockOnTransition2: any, + mockOnTransition3: any, context?: any, onPostTransition?: any ) => { @@ -21,17 +21,17 @@ const getTestDefinition = ( context, states: { state1: { - onTransitionTo: mockEvent1, + onTransition: mockOnTransition1, onPostTransition, nextState: 'state2', }, state2: { - onTransitionTo: mockEvent2, + onTransition: mockOnTransition2, onPostTransition, nextState: 'state3', }, state3: { - onTransitionTo: mockEvent3, + onTransition: mockOnTransition3, onPostTransition, nextState: 'end', }, @@ -52,15 +52,19 @@ describe('handleState', () => { }); it('should execute all the state machine transitions based on the provided data structure', async () => { - const mockEventState1 = jest.fn(); - const mockEventState2 = jest.fn(); - const mockEventState3 = jest.fn(); - const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); + const mockOnTransitionState1 = jest.fn(); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3 + ); await handleState('state1', testDefinition, testDefinition.context); - expect(mockEventState1).toHaveBeenCalledTimes(1); - expect(mockEventState2).toHaveBeenCalledTimes(1); - expect(mockEventState3).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(1); expect(mockContract.logger?.debug).toHaveBeenCalledWith( 'Executed state: state1 with status: success - nextState: state2' ); @@ -73,54 +77,62 @@ describe('handleState', () => { }); it('should execute the transition from the provided state', async () => { - const mockEventState1 = jest.fn(); - const mockEventState2 = jest.fn(); - const mockEventState3 = jest.fn(); - const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); + const mockOnTransitionState1 = jest.fn(); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3 + ); await handleState('state2', testDefinition, testDefinition.context); - expect(mockEventState1).toHaveBeenCalledTimes(0); - expect(mockEventState2).toHaveBeenCalledTimes(1); - expect(mockEventState3).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState1).toHaveBeenCalledTimes(0); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(1); }); it('should exit when a state returns error', async () => { const error = new Error('Installation failed'); - const mockEventState1 = jest.fn().mockRejectedValue(error); - const mockEventState2 = jest.fn(); - const mockEventState3 = jest.fn(); - const testDefinition = getTestDefinition(mockEventState1, mockEventState2, mockEventState3); + const mockOnTransitionState1 = jest.fn().mockRejectedValue(error); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3 + ); await handleState('state1', testDefinition, testDefinition.context); - expect(mockEventState1).toHaveBeenCalledTimes(1); - expect(mockEventState2).toHaveBeenCalledTimes(0); - expect(mockEventState3).toHaveBeenCalledTimes(0); + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(0); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); expect(mockContract.logger?.warn).toHaveBeenCalledWith( 'Error during execution of state "state1" with status "failed": Installation failed' ); }); it('should call the onTransition function with context data and the return value is saved for the next iteration', async () => { - const mockEventState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); - const mockEventState2 = jest + const mockOnTransitionState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); + const mockOnTransitionState2 = jest .fn() .mockImplementation(() => Promise.resolve({ promiseData: {} })); - const mockEventState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); const contextData = { testData: 'test' }; const testDefinition = getTestDefinition( - mockEventState1, - mockEventState2, - mockEventState3, + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, contextData ); await handleState('state1', testDefinition, testDefinition.context); - expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockEventState2).toHaveBeenCalledWith({ + expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith({ testData: 'test', arrayData: ['test1', 'test2'], }); - expect(mockEventState3).toHaveBeenCalledWith({ + expect(mockOnTransitionState3).toHaveBeenCalledWith({ testData: 'test', arrayData: ['test1', 'test2'], promiseData: {}, @@ -138,32 +150,32 @@ describe('handleState', () => { }); it('should save the return data from transitions also when return type is function', async () => { - const mockEventState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); + const mockOnTransitionState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); const state2Result = () => { return { innerData: 'test', }; }; - const mockEventState2 = jest.fn().mockImplementation(() => { + const mockOnTransitionState2 = jest.fn().mockImplementation(() => { return state2Result; }); - const mockEventState3 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); const contextData = { testData: 'test' }; const testDefinition = getTestDefinition( - mockEventState1, - mockEventState2, - mockEventState3, + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, contextData ); await handleState('state1', testDefinition, testDefinition.context); - expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockEventState2).toHaveBeenCalledWith({ + expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith({ testData: 'test', arrayData: ['test1', 'test2'], state2Result, }); - expect(mockEventState3).toHaveBeenCalledWith({ + expect(mockOnTransitionState3).toHaveBeenCalledWith({ testData: 'test', arrayData: ['test1', 'test2'], state2Result, @@ -181,7 +193,7 @@ describe('handleState', () => { }); it('should return updated context data', async () => { - const mockEventState1 = jest + const mockOnTransitionState1 = jest .fn() .mockImplementation(() => Promise.resolve({ promiseData: {} })); const state2Result = () => { @@ -189,26 +201,26 @@ describe('handleState', () => { innerData: 'test', }; }; - const mockEventState2 = jest.fn().mockImplementation(() => { + const mockOnTransitionState2 = jest.fn().mockImplementation(() => { return state2Result; }); - const mockEventState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); const contextData = { testData: 'test' }; const testDefinition = getTestDefinition( - mockEventState1, - mockEventState2, - mockEventState3, + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, contextData ); const updatedContext = await handleState('state1', testDefinition, testDefinition.context); - expect(mockEventState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockEventState2).toHaveBeenCalledWith({ + expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith({ testData: 'test', promiseData: {}, state2Result, }); - expect(mockEventState3).toHaveBeenCalledWith({ + expect(mockOnTransitionState3).toHaveBeenCalledWith({ testData: 'test', promiseData: {}, state2Result, @@ -222,25 +234,52 @@ describe('handleState', () => { }); }); + it('should update a variable in the context at every call and return the updated value', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ runningVal: 'test1' }); + const mockOnTransitionState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ runningVal: 'test2' })); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ runningVal: 'test3' }); + const contextData = { runningVal: [], fixedVal: 'something' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + expect(mockOnTransitionState1).toHaveBeenCalledWith({ runningVal: [], fixedVal: 'something' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith({ + runningVal: 'test1', + fixedVal: 'something', + }); + expect(mockOnTransitionState3).toHaveBeenCalledWith({ + runningVal: 'test2', + fixedVal: 'something', + }); + expect(updatedContext).toEqual({ fixedVal: 'something', runningVal: 'test3' }); + }); + it('should execute postTransition function after the transition is complete', async () => { - const mockEventState1 = jest.fn(); - const mockEventState2 = jest.fn(); - const mockEventState3 = jest.fn(); + const mockOnTransitionState1 = jest.fn(); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); const mockPostTransition = jest.fn(); const testDefinition = getTestDefinition( - mockEventState1, - mockEventState2, - mockEventState3, + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, undefined, mockPostTransition ); await handleState('state1', testDefinition, testDefinition.context); - expect(mockEventState1).toHaveBeenCalled(); + expect(mockOnTransitionState1).toHaveBeenCalled(); expect(mockPostTransition).toHaveBeenCalled(); - expect(mockEventState2).toHaveBeenCalled(); + expect(mockOnTransitionState2).toHaveBeenCalled(); expect(mockPostTransition).toHaveBeenCalled(); - expect(mockEventState3).toHaveBeenCalled(); + expect(mockOnTransitionState3).toHaveBeenCalled(); expect(mockPostTransition).toHaveBeenCalled(); expect(mockContract.logger?.debug).toHaveBeenCalledWith( 'Executing post transition function: mockConstructor' diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 9c9300daeb8cb..95af07f61c1f4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -6,7 +6,7 @@ */ import { appContextService } from '../../app_context'; export interface State { - onTransitionTo: any; + onTransition: any; nextState?: string; currentStatus?: string; onPostTransition?: any; @@ -30,9 +30,9 @@ export async function handleState( let currentStatus = 'pending'; let stateResult; let updatedContext = { ...context }; - if (typeof currentState.onTransitionTo === 'function') { + if (typeof currentState.onTransition === 'function') { try { - stateResult = await currentState.onTransitionTo.call(undefined, updatedContext); + stateResult = await currentState.onTransition.call(undefined, updatedContext); // check if is a function/promise if (typeof stateResult === 'function') { const promiseName = `${currentStateName}Result`; From 25c5ea52c648d8c3978a94bd8cde66140f5c7f75 Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 22 Mar 2024 12:04:05 +0100 Subject: [PATCH 12/38] Add feature flag and plug new functions --- .../fleet/common/experimental_features.ts | 1 + .../plugins/fleet/common/types/models/epm.ts | 1 + .../fleet/common/types/rest_spec/epm.ts | 3 +- .../_state_machine_package_install.ts | 18 +- .../server/services/epm/packages/install.ts | 238 ++++++++++++++++-- .../services/epm/packages/install_steps.ts | 22 +- x-pack/plugins/fleet/server/types/index.tsx | 1 + 7 files changed, 250 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index 8271f0403beda..44421348e3d72 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -28,6 +28,7 @@ export const allowedExperimentalValues = Object.freeze>( agentless: false, enableStrictKQLValidation: false, subfeaturePrivileges: false, + enablePackagesStateMachine: false, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index b1bea249ee9de..cca3ede2595ac 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -37,6 +37,7 @@ export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'in export type InstallSource = 'registry' | 'upload' | 'bundled' | 'custom'; export type EpmPackageInstallStatus = 'installed' | 'installing' | 'install_failed'; +export type InstallResultStatus = 'installed' | 'already_installed'; export type ServiceName = 'kibana' | 'elasticsearch'; export type AgentAssetType = typeof agentAssetTypes; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts index 7300bd5449333..4882c1c0652e6 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts @@ -18,6 +18,7 @@ import type { EpmPackageInstallStatus, SimpleSOAssetType, AssetSOObject, + InstallResultStatus, } from '../models/epm'; export interface GetCategoriesRequest { @@ -154,7 +155,7 @@ export interface IBulkInstallPackageHTTPError { export interface InstallResult { assets?: AssetReference[]; - status?: 'installed' | 'already_installed'; + status?: InstallResultStatus; error?: Error; installType: InstallType; installSource: InstallSource; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts index 59c5308996bcb..967bec2063d37 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts @@ -26,6 +26,7 @@ import type { EsAssetReference, KibanaAssetReference, IndexTemplateEntry, + AssetReference, } from '../../../types'; import { @@ -79,14 +80,16 @@ export interface InstallContext { ignoreMappingUpdateErrors?: boolean; skipDataStreamRollover?: boolean; - indexTemplates: IndexTemplateEntry[]; - packageAssetRefs: PackageAssetReference[]; + indexTemplates?: IndexTemplateEntry[]; + packageAssetRefs?: PackageAssetReference[]; // output values - esReferences: EsAssetReference[]; - kibanaAssetPromise: Promise; + esReferences?: EsAssetReference[]; + kibanaAssetPromise?: Promise; } -export async function _stateMachineInstallPackage({ context }: { context: InstallContext }) { +export async function _stateMachineInstallPackage( + context: InstallContext +): Promise { const installStates: StateMachineDefinition = { context, states: { @@ -141,5 +144,8 @@ export async function _stateMachineInstallPackage({ context }: { context: Instal installStates, installStates.context ); - return [...installedKibanaAssetsRefs, ...esReferences]; + return [ + ...(installedKibanaAssetsRefs as KibanaAssetReference[]), + ...(esReferences as EsAssetReference[]), + ]; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 43b0c9d68a04c..6e2e37f20417a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -38,6 +38,7 @@ import type { NewPackagePolicy, PackageInfo, PackageVerificationResult, + InstallResultStatus, } from '../../../types'; import { AUTO_UPGRADE_POLICIES_PACKAGES, @@ -70,6 +71,8 @@ import { sendTelemetryEvents, UpdateEventType } from '../../upgrade_sender'; import { auditLoggingService } from '../../audit_logging'; import { getFilteredInstallPackages } from '../filtered_packages'; +import { _stateMachineInstallPackage } from './_state_machine_package_install'; + import { formatVerificationResultForSO } from './package_verification'; import { getInstallation, getInstallationObject } from './get'; import { removeInstallation } from './remove'; @@ -458,24 +461,226 @@ async function installPackageFromRegistry({ }` ); } - - return await installPackageCommon({ - pkgName, - pkgVersion, + const { enablePackagesStateMachine } = appContextService.getExperimentalFeatures(); + if (enablePackagesStateMachine) { + return await installPackageWitStateMachine({ + pkgName, + pkgVersion, + installSource, + installedPkg, + installType, + savedObjectsClient, + esClient, + spaceId, + force, + packageInstallContext, + paths, + verificationResult, + authorizationHeader, + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }); + } else { + return await installPackageCommon({ + pkgName, + pkgVersion, + installSource, + installedPkg, + installType, + savedObjectsClient, + esClient, + spaceId, + force, + packageInstallContext, + paths, + verificationResult, + authorizationHeader, + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }); + } + } catch (e) { + sendEvent({ + ...telemetryEvent, + errorMessage: e.message, + }); + return { + error: e, + installType, installSource, - installedPkg, + }; + } +} + +function getElasticSubscription(packageInfo: ArchivePackage) { + const subscription = packageInfo.conditions?.elastic?.subscription as LicenseType | undefined; + // Keep packageInfo.license for backward compatibility + return subscription || packageInfo.license || 'basic'; +} + +async function installPackageCommon(options: { + pkgName: string; + pkgVersion: string; + installSource: InstallSource; + installedPkg?: SavedObject; + installType: InstallType; + savedObjectsClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; + spaceId: string; + force?: boolean; + packageInstallContext: PackageInstallContext; + paths: string[]; + verificationResult?: PackageVerificationResult; + telemetryEvent?: PackageUpdateEvent; + authorizationHeader?: HTTPAuthorizationHeader | null; + ignoreMappingUpdateErrors?: boolean; + skipDataStreamRollover?: boolean; +}): Promise { + const packageInfo = options.packageInstallContext.packageInfo; + + const { + pkgName, + pkgVersion, + installSource, + installedPkg, + installType, + savedObjectsClient, + force, + esClient, + spaceId, + verificationResult, + authorizationHeader, + ignoreMappingUpdateErrors, + skipDataStreamRollover, + packageInstallContext, + } = options; + let { telemetryEvent } = options; + const logger = appContextService.getLogger(); + logger.info(`Install - Starting installation of ${pkgName}@${pkgVersion} from ${installSource} `); + + // Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611 + await Promise.resolve(); + const span = apm.startSpan( + `Install package from ${installSource} ${pkgName}@${pkgVersion}`, + 'package' + ); + + if (!telemetryEvent) { + telemetryEvent = getTelemetryEvent(pkgName, pkgVersion); + telemetryEvent.installType = installType; + telemetryEvent.currentVersion = installedPkg?.attributes.version || 'not_installed'; + } + + try { + span?.addLabels({ + packageName: pkgName, + packageVersion: pkgVersion, installType, + }); + + const filteredPackages = getFilteredInstallPackages(); + if (filteredPackages.includes(pkgName)) { + throw new FleetUnauthorizedError(`${pkgName} installation is not authorized`); + } + + // if the requested version is the same as installed version, check if we allow it based on + // current installed package status and force flag, if we don't allow it, + // just return the asset references from the existing installation + if ( + installedPkg?.attributes.version === pkgVersion && + installedPkg?.attributes.install_status === 'installed' + ) { + if (!force) { + logger.debug(`${pkgName}-${pkgVersion} is already installed, skipping installation`); + return { + assets: [ + ...installedPkg.attributes.installed_es, + ...installedPkg.attributes.installed_kibana, + ], + status: 'already_installed', + installType, + installSource, + }; + } + } + const elasticSubscription = getElasticSubscription(packageInfo); + if (!licenseService.hasAtLeast(elasticSubscription)) { + logger.error(`Installation requires ${elasticSubscription} license`); + const err = new FleetError(`Installation requires ${elasticSubscription} license`); + sendEvent({ + ...telemetryEvent, + errorMessage: err.message, + }); + return { error: err, installType, installSource }; + } + + // Saved object client need to be scopped with the package space for saved object tagging + const savedObjectClientWithSpace = appContextService.getInternalUserSOClientForSpaceId(spaceId); + + const savedObjectsImporter = appContextService + .getSavedObjects() + .createImporter(savedObjectClientWithSpace, { importSizeLimit: 15_000 }); + + const savedObjectTagAssignmentService = appContextService + .getSavedObjectsTagging() + .createInternalAssignmentService({ client: savedObjectClientWithSpace }); + + const savedObjectTagClient = appContextService + .getSavedObjectsTagging() + .createTagClient({ client: savedObjectClientWithSpace }); + + // try installing the package, if there was an error, call error handler and rethrow + return await _installPackage({ savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, esClient, - spaceId, - force, + logger, + installedPkg, packageInstallContext, - paths, + installType, + spaceId, verificationResult, + installSource, authorizationHeader, + force, ignoreMappingUpdateErrors, skipDataStreamRollover, - }); + }) + .then(async (assets) => { + logger.debug(`Removing old assets from previous versions of ${pkgName}`); + await removeOldAssets({ + soClient: savedObjectsClient, + pkgName: packageInfo.name, + currentVersion: packageInfo.version, + }); + sendEvent({ + ...telemetryEvent!, + status: 'success', + }); + return { assets, status: 'installed' as InstallResultStatus, installType, installSource }; + }) + .catch(async (err: Error) => { + logger.warn(`Failure to install package [${pkgName}]: [${err.toString()}]`, { + error: { stack_trace: err.stack }, + }); + await handleInstallPackageFailure({ + savedObjectsClient, + error: err, + pkgName, + pkgVersion, + installedPkg, + spaceId, + esClient, + authorizationHeader, + }); + sendEvent({ + ...telemetryEvent!, + errorMessage: err.message, + }); + return { error: err, installType, installSource }; + }); } catch (e) { sendEvent({ ...telemetryEvent, @@ -486,16 +691,12 @@ async function installPackageFromRegistry({ installType, installSource, }; + } finally { + span?.end(); } } -function getElasticSubscription(packageInfo: ArchivePackage) { - const subscription = packageInfo.conditions?.elastic?.subscription as LicenseType | undefined; - // Keep packageInfo.license for backward compatibility - return subscription || packageInfo.license || 'basic'; -} - -async function installPackageCommon(options: { +async function installPackageWitStateMachine(options: { pkgName: string; pkgVersion: string; installSource: InstallSource; @@ -607,8 +808,7 @@ async function installPackageCommon(options: { .createTagClient({ client: savedObjectClientWithSpace }); // try installing the package, if there was an error, call error handler and rethrow - // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' - return await _installPackage({ + return await _stateMachineInstallPackage({ savedObjectsClient, savedObjectsImporter, savedObjectTagAssignmentService, @@ -637,7 +837,7 @@ async function installPackageCommon(options: { ...telemetryEvent!, status: 'success', }); - return { assets, status: 'installed', installType, installSource }; + return { assets, status: 'installed' as InstallResultStatus, installType, installSource }; }) .catch(async (err: Error) => { logger.warn(`Failure to install package [${pkgName}]: [${err.toString()}]`, { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts index d37f74abdb3c4..6ca95f3988f4f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts @@ -162,7 +162,13 @@ export async function stepInstallILMPolicies({ context }: { context: InstallCont appContextService.getConfig()?.internal?.disableILMPolicies ?? false; if (!isILMPoliciesDisabled) { updatedEsReferences = await withPackageSpan('Install ILM policies', () => - installILMPolicy(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) + installILMPolicy( + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences || [] + ) ); logger.debug(`Package install - Installing Data Stream ILM policies`); ({ esReferences: updatedEsReferences } = await withPackageSpan( @@ -173,7 +179,7 @@ export async function stepInstallILMPolicies({ context }: { context: InstallCont esClient, savedObjectsClient, logger, - esReferences + esReferences || [] ) )); } @@ -186,7 +192,7 @@ export async function stepInstallMlModel({ context }: { context: InstallContext // installs ml models logger.debug(`Package install - installing ML models`); const updatedEsReferences = await withPackageSpan('Install ML models', () => - installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) + installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences || []) ); return { esReferences: updatedEsReferences }; } @@ -213,7 +219,7 @@ export async function stepInstallIndexTemplatePipelines({ context }: { context: esClient, savedObjectsClient, logger, - esReferences, + esReferences: esReferences || [], }); return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; } @@ -243,7 +249,7 @@ export async function stepInstallIndexTemplatePipelines({ context }: { context: esClient, savedObjectsClient, logger, - esReferences, + esReferences: esReferences || [], onlyForDataStreams: dataStreams, }); return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; @@ -269,7 +275,7 @@ export async function stepUpdateCurrentWriteIndices({ context }: { context: Inst // update current backing indices of each data stream logger.debug(`Package install - Updating backing indices of each data stream`); await withPackageSpan('Update write indices', () => - updateCurrentWriteIndices(esClient, logger, indexTemplates, { + updateCurrentWriteIndices(esClient, logger, indexTemplates || [], { ignoreMappingUpdateErrors, skipDataStreamRollover, }) @@ -331,7 +337,7 @@ export async function stepDeletePreviousPipelines({ context }: { context: Instal savedObjectsClient, pkgName, installedPkg!.attributes.version, - esReferences + esReferences || [] ) ); } @@ -344,7 +350,7 @@ export async function stepDeletePreviousPipelines({ context }: { context: Instal savedObjectsClient, pkgName, installedPkg!.attributes.install_version, - esReferences + esReferences || [] ) ); } diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index bec381d311937..18a3ed05bfcd2 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -97,6 +97,7 @@ export type { ActionStatusOptions, PackageSpecTags, AssetsMap, + InstallResultStatus, } from '../../common/types'; export { ElasticsearchAssetType, KibanaAssetType, KibanaSavedObjectType } from '../../common/types'; export { dataTypes } from '../../common/constants'; From c3d553731852d322d6e0da3fab22706aa69d6645 Mon Sep 17 00:00:00 2001 From: criamico Date: Mon, 25 Mar 2024 17:04:21 +0100 Subject: [PATCH 13/38] Fix error in header functions and add logging --- .../_state_machine_package_install.ts | 15 +++++++++---- .../server/services/epm/packages/install.ts | 4 +++- .../services/epm/packages/install_steps.ts | 22 +++++++++---------- .../packages/integrations_state_machine.ts | 10 ++++++--- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts index 967bec2063d37..c20b3576a868f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts @@ -144,8 +144,15 @@ export async function _stateMachineInstallPackage( installStates, installStates.context ); - return [ - ...(installedKibanaAssetsRefs as KibanaAssetReference[]), - ...(esReferences as EsAssetReference[]), - ]; + if ( + installedKibanaAssetsRefs && + installedKibanaAssetsRefs.length && + esReferences && + esReferences.length + ) + return [ + ...(installedKibanaAssetsRefs as KibanaAssetReference[]), + ...(esReferences as EsAssetReference[]), + ]; + return []; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 6e2e37f20417a..b3419469ebcc5 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -734,7 +734,9 @@ async function installPackageWitStateMachine(options: { } = options; let { telemetryEvent } = options; const logger = appContextService.getLogger(); - logger.info(`Install - Starting installation of ${pkgName}@${pkgVersion} from ${installSource} `); + logger.info( + `Install with enablePackagesStateMachine - Starting installation of ${pkgName}@${pkgVersion} from ${installSource} ` + ); // Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611 await Promise.resolve(); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts index 6ca95f3988f4f..c0243427f2ac2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts @@ -42,7 +42,7 @@ import { withPackageSpan } from './utils'; import type { InstallContext } from './_state_machine_package_install'; import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; -export async function stepCreateRestartInstallation({ context }: { context: InstallContext }) { +export async function stepCreateRestartInstallation(context: InstallContext) { const { savedObjectsClient, logger, @@ -117,7 +117,7 @@ export async function stepCreateRestartInstallation({ context }: { context: Inst return { esReferences }; } -export async function stepInstallKibanaAssets({ context }: { context: InstallContext }) { +export async function stepInstallKibanaAssets(context: InstallContext) { const { savedObjectsClient, savedObjectsImporter, @@ -152,7 +152,7 @@ export async function stepInstallKibanaAssets({ context }: { context: InstallCon return { kibanaAssetPromise }; } -export async function stepInstallILMPolicies({ context }: { context: InstallContext }) { +export async function stepInstallILMPolicies(context: InstallContext) { const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; let updatedEsReferences; // currently only the base package has an ILM policy @@ -186,7 +186,7 @@ export async function stepInstallILMPolicies({ context }: { context: InstallCont return { esReferences: updatedEsReferences }; } -export async function stepInstallMlModel({ context }: { context: InstallContext }) { +export async function stepInstallMlModel(context: InstallContext) { const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; // installs ml models @@ -197,7 +197,7 @@ export async function stepInstallMlModel({ context }: { context: InstallContext return { esReferences: updatedEsReferences }; } -export async function stepInstallIndexTemplatePipelines({ context }: { context: InstallContext }) { +export async function stepInstallIndexTemplatePipelines(context: InstallContext) { const { esClient, savedObjectsClient, @@ -257,7 +257,7 @@ export async function stepInstallIndexTemplatePipelines({ context }: { context: } } -export async function stepRemoveLegacyTemplates({ context }: { context: InstallContext }) { +export async function stepRemoveLegacyTemplates(context: InstallContext) { const { esClient, packageInstallContext, logger } = context; const { packageInfo } = packageInstallContext; try { @@ -268,7 +268,7 @@ export async function stepRemoveLegacyTemplates({ context }: { context: InstallC } } -export async function stepUpdateCurrentWriteIndices({ context }: { context: InstallContext }) { +export async function stepUpdateCurrentWriteIndices(context: InstallContext) { const { esClient, logger, ignoreMappingUpdateErrors, skipDataStreamRollover, indexTemplates } = context; @@ -282,7 +282,7 @@ export async function stepUpdateCurrentWriteIndices({ context }: { context: Inst ); } -export async function stepInstallTransforms({ context }: { context: InstallContext }) { +export async function stepInstallTransforms(context: InstallContext) { const { packageInstallContext, esClient, @@ -309,7 +309,7 @@ export async function stepInstallTransforms({ context }: { context: InstallConte return { esReferences: res.esReferences }; } -export async function stepDeletePreviousPipelines({ context }: { context: InstallContext }) { +export async function stepDeletePreviousPipelines(context: InstallContext) { const { packageInstallContext, esClient, @@ -357,7 +357,7 @@ export async function stepDeletePreviousPipelines({ context }: { context: Instal return { esReferences: updatedESReferences }; } -export async function stepSaveArchiveEntries({ context }: { context: InstallContext }) { +export async function stepSaveArchiveEntries(context: InstallContext) { const { packageInstallContext, savedObjectsClient, logger, installSource, kibanaAssetPromise } = context; const installedKibanaAssetsRefs = await kibanaAssetPromise; @@ -383,7 +383,7 @@ export async function stepSaveArchiveEntries({ context }: { context: InstallCont return { packageAssetRefs, installedKibanaAssetsRefs }; } -export async function stepSaveSystemObject({ context }: { context: InstallContext }) { +export async function stepSaveSystemObject(context: InstallContext) { const { packageInstallContext, savedObjectsClient, logger, esClient } = context; const { packageInfo } = packageInstallContext; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 95af07f61c1f4..c191689094a5d 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -31,6 +31,9 @@ export async function handleState( let stateResult; let updatedContext = { ...context }; if (typeof currentState.onTransition === 'function') { + logger.debug( + `Current state ${currentStateName} - Running transition ${currentState.onTransition.name}` + ); try { stateResult = await currentState.onTransition.call(undefined, updatedContext); // check if is a function/promise @@ -41,6 +44,9 @@ export async function handleState( updatedContext = { ...updatedContext, ...stateResult }; } currentStatus = 'success'; + logger.debug( + `Executed state: ${currentStateName} with status: ${currentStatus} - nextState: ${currentState.nextState}` + ); } catch (error) { currentStatus = 'failed'; logger.warn( @@ -50,9 +56,7 @@ export async function handleState( } else { currentStatus = 'failed'; } - logger.debug( - `Executed state: ${currentStateName} with status: ${currentStatus} - nextState: ${currentState.nextState}` - ); + if (typeof currentState.onPostTransition === 'function') { try { await currentState.onPostTransition.call(undefined, updatedContext); From c6f080fdecb0f0c6ea89a07cead4b402d27f1881 Mon Sep 17 00:00:00 2001 From: criamico Date: Tue, 26 Mar 2024 12:04:13 +0100 Subject: [PATCH 14/38] Fix errors in install process and modify logging --- .../services/epm/packages/_install_package.ts | 2 +- .../services/epm/packages/install_steps.ts | 81 ++++++++++--------- .../packages/integrations_state_machine.ts | 2 +- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 33fa489c81466..8b8a44b55e222 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -97,8 +97,8 @@ export async function _installPackage({ const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo; try { + // if some installation already exists if (installedPkg) { - // if some installation already exists const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; const hasExceededTimeout = Date.now() - Date.parse(installedPkg.attributes.install_started_at) < diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts index c0243427f2ac2..6d74474d4495b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts @@ -12,8 +12,9 @@ import { PACKAGES_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT, + FLEET_INSTALL_FORMAT_VERSION, } from '../../../constants'; -import type { PackageAssetReference, Installation } from '../../../types'; +import type { PackageAssetReference, Installation, EsAssetReference } from '../../../types'; import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; @@ -41,6 +42,7 @@ import { restartInstallation, createInstallation } from './install'; import { withPackageSpan } from './utils'; import type { InstallContext } from './_state_machine_package_install'; import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; +import { clearLatestFailedAttempts } from './install_errors_helpers'; export async function stepCreateRestartInstallation(context: InstallContext) { const { @@ -131,8 +133,6 @@ export async function stepInstallKibanaAssets(context: InstallContext) { const { packageInfo, paths } = packageInstallContext; const { name: pkgName, title: pkgTitle } = packageInfo; - logger.debug(`Package install - Installing Kibana assets`); - // step install_kibana_assets const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => installKibanaAssetsAndReferences({ savedObjectsClient, @@ -154,7 +154,8 @@ export async function stepInstallKibanaAssets(context: InstallContext) { export async function stepInstallILMPolicies(context: InstallContext) { const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; - let updatedEsReferences; + let updatedEsReferences: EsAssetReference[] = []; + // currently only the base package has an ILM policy // at some point ILM policies can be installed/modified // per data stream and we should then save them @@ -170,27 +171,23 @@ export async function stepInstallILMPolicies(context: InstallContext) { esReferences || [] ) ); - logger.debug(`Package install - Installing Data Stream ILM policies`); - ({ esReferences: updatedEsReferences } = await withPackageSpan( - 'Install Data Stream ILM policies', - () => - installIlmForDataStream( - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences || [] - ) - )); + + const res = await withPackageSpan('Install Data Stream ILM policies', () => + installIlmForDataStream( + packageInstallContext, + esClient, + savedObjectsClient, + logger, + updatedEsReferences + ) + ); + return { esReferences: res.esReferences }; } - return { esReferences: updatedEsReferences }; } export async function stepInstallMlModel(context: InstallContext) { const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; - // installs ml models - logger.debug(`Package install - installing ML models`); const updatedEsReferences = await withPackageSpan('Install ML models', () => installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences || []) ); @@ -209,9 +206,6 @@ export async function stepInstallIndexTemplatePipelines(context: InstallContext) const { packageInfo } = packageInstallContext; if (packageInfo.type === 'integration') { - logger.debug( - `Package install - Installing index templates and pipelines, packageInfo.type ${packageInfo.type}` - ); const { installedTemplates, esReferences: templateEsReferences } = await installIndexTemplatesAndPipelines({ installedPkg: installedPkg ? installedPkg.attributes : undefined, @@ -239,9 +233,6 @@ export async function stepInstallIndexTemplatePipelines(context: InstallContext) ); if (dataStreams.length) { - logger.debug( - `Package install - installing index templates and pipelines with datastreams length ${dataStreams.length}` - ); const { installedTemplates, esReferences: templateEsReferences } = await installIndexTemplatesAndPipelines({ installedPkg: installedPkg ? installedPkg.attributes : undefined, @@ -261,7 +252,6 @@ export async function stepRemoveLegacyTemplates(context: InstallContext) { const { esClient, packageInstallContext, logger } = context; const { packageInfo } = packageInstallContext; try { - logger.debug(`Package install - Removing legacy templates`); await removeLegacyTemplates({ packageInfo, esClient, logger }); } catch (e) { logger.warn(`Error removing legacy templates: ${e.message}`); @@ -273,7 +263,6 @@ export async function stepUpdateCurrentWriteIndices(context: InstallContext) { context; // update current backing indices of each data stream - logger.debug(`Package install - Updating backing indices of each data stream`); await withPackageSpan('Update write indices', () => updateCurrentWriteIndices(esClient, logger, indexTemplates || [], { ignoreMappingUpdateErrors, @@ -292,8 +281,7 @@ export async function stepInstallTransforms(context: InstallContext) { force, authorizationHeader, } = context; - logger.debug(`Package install - Installing transforms`); - // steps install_transforms + const res = await withPackageSpan('Install transforms', () => installTransforms({ packageInstallContext, @@ -358,12 +346,11 @@ export async function stepDeletePreviousPipelines(context: InstallContext) { } export async function stepSaveArchiveEntries(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, logger, installSource, kibanaAssetPromise } = - context; + const { packageInstallContext, savedObjectsClient, installSource, kibanaAssetPromise } = context; const installedKibanaAssetsRefs = await kibanaAssetPromise; const { packageInfo } = packageInstallContext; - logger.debug(`Package install - Updating archive entries`); + const packageAssetResults = await withPackageSpan('Update archive entries', () => saveArchiveEntriesFromAssetsMap({ savedObjectsClient, @@ -384,10 +371,16 @@ export async function stepSaveArchiveEntries(context: InstallContext) { } export async function stepSaveSystemObject(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, logger, esClient } = context; - + const { + packageInstallContext, + savedObjectsClient, + logger, + esClient, + installedPkg, + packageAssetRefs, + } = context; const { packageInfo } = packageInstallContext; - const { name: pkgName } = packageInfo; + const { name: pkgName, version: pkgVersion } = packageInfo; auditLoggingService.writeCustomSoAuditLog({ action: 'update', @@ -395,6 +388,20 @@ export async function stepSaveSystemObject(context: InstallContext) { savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, }); + await withPackageSpan('Update install status', () => + savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + version: pkgVersion, + install_version: pkgVersion, + install_status: 'installed', + package_assets: packageAssetRefs, + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, + latest_install_failed_attempts: clearLatestFailedAttempts( + pkgVersion, + installedPkg?.attributes.latest_install_failed_attempts ?? [] + ), + }) + ); + // Need to refetch the installation again to retrieve all the attributes const updatedPackage = await savedObjectsClient.get( PACKAGES_SAVED_OBJECT_TYPE, @@ -416,5 +423,7 @@ export async function stepSaveSystemObject(context: InstallContext) { await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); }); } - logger.debug(`Package install - Installation complete`); + logger.debug( + `Install status ${updatedPackage?.attributes?.install_status} - Installation complete!` + ); } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index c191689094a5d..dc68cc2886dbe 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -32,7 +32,7 @@ export async function handleState( let updatedContext = { ...context }; if (typeof currentState.onTransition === 'function') { logger.debug( - `Current state ${currentStateName} - Running transition ${currentState.onTransition.name}` + `Current state ${currentStateName}: running transition ${currentState.onTransition.name}` ); try { stateResult = await currentState.onTransition.call(undefined, updatedContext); From 807432899224b4320bef0e1c109bb5f048eeb6ec Mon Sep 17 00:00:00 2001 From: criamico Date: Tue, 26 Mar 2024 17:07:20 +0100 Subject: [PATCH 15/38] Fix unit tests and add feature flag check --- .../services/epm/packages/install.test.ts | 644 +++++++++++++----- 1 file changed, 455 insertions(+), 189 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index 97817b063b730..921dbe63809cb 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -29,6 +29,7 @@ import { isPackageVersionOrLaterInstalled, } from './install'; import * as install from './_install_package'; +import * as installStateMachine from './_state_machine_package_install'; import { getBundledPackageByPkgKey } from './bundled_packages'; import { getInstalledPackageWithAssets, getInstallationObject } from './get'; @@ -57,6 +58,7 @@ jest.mock('../../app_context', () => { getConfig: jest.fn(() => ({})), getSavedObjectsTagging: jest.fn(() => mockedSavedObjectTagging), getInternalUserSOClientForSpaceId: jest.fn(), + getExperimentalFeatures: jest.fn(), }, }; }); @@ -79,6 +81,11 @@ jest.mock('./_install_package', () => { _installPackage: jest.fn(() => Promise.resolve()), }; }); +jest.mock('./_state_machine_package_install', () => { + return { + _stateMachineInstallPackage: jest.fn(() => Promise.resolve()), + }; +}); jest.mock('../kibana/index_pattern/install', () => { return { installIndexPatterns: jest.fn(() => Promise.resolve()), @@ -161,246 +168,504 @@ describe('install', () => { jest.mocked(Registry.getPackage).mockImplementation(() => Promise.resolve({ packageInfo: { license: 'basic', conditions: { elastic: { subscription: 'basic' } } }, + paths: [], } as any) ); mockGetBundledPackageByPkgKey.mockReset(); (install._installPackage as jest.Mock).mockClear(); + (installStateMachine._stateMachineInstallPackage as jest.Mock).mockClear(); jest.mocked(appContextService.getInternalUserSOClientForSpaceId).mockReset(); }); describe('registry', () => { - beforeEach(() => { - mockGetBundledPackageByPkgKey.mockResolvedValue(undefined); - }); - - it('should send telemetry on install failure, out of date', async () => { - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.1.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + describe('with enablePackagesStateMachine = false', () => { + beforeEach(() => { + mockGetBundledPackageByPkgKey.mockResolvedValue(undefined); + jest.mocked(appContextService.getExperimentalFeatures).mockReturnValue({ + enablePackagesStateMachine: false, + } as any); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - errorMessage: 'apache-1.1.0 is out-of-date and cannot be installed or updated', - eventType: 'package-install', - installType: 'install', - newVersion: '1.1.0', - packageName: 'apache', - status: 'failure', + it('should send telemetry on install failure, out of date', async () => { + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.1.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'apache-1.1.0 is out-of-date and cannot be installed or updated', + eventType: 'package-install', + installType: 'install', + newVersion: '1.1.0', + packageName: 'apache', + status: 'failure', + }); }); - }); - it('should send telemetry on install failure, license error', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on install failure, license error', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'Installation requires basic license', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - errorMessage: 'Installation requires basic license', - eventType: 'package-install', - installType: 'install', - newVersion: '1.3.0', - packageName: 'apache', - status: 'failure', + it('should send telemetry on install success', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'success', + }); }); - }); - it('should send telemetry on install success', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on update success', async () => { + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: '1.2.0', + dryRun: false, + eventType: 'package-install', + installType: 'update', + newVersion: '1.3.0', + packageName: 'apache', + status: 'success', + }); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - eventType: 'package-install', - installType: 'install', - newVersion: '1.3.0', - packageName: 'apache', - status: 'success', + it('should send telemetry on install failure, async error', async () => { + jest.mocked(install._installPackage).mockRejectedValue(new Error('error')); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'error', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - }); - it('should send telemetry on update success', async () => { - jest - .mocked(getInstallationObject) - .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + it('should install from bundled package if one exists', async () => { + (install._installPackage as jest.Mock).mockResolvedValue({}); + mockGetBundledPackageByPkgKey.mockResolvedValue({ + name: 'test_package', + version: '1.0.0', + getBuffer: async () => Buffer.from('test_package'), + }); + + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package-1.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.error).toBeUndefined(); + + expect(install._installPackage).toHaveBeenCalledWith( + expect.objectContaining({ installSource: 'bundled' }) + ); + }); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should fetch latest version if version not provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('installed'); + + expect(sendTelemetryEvents).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.objectContaining({ + newVersion: '1.3.0', + }) + ); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: '1.2.0', - dryRun: false, - eventType: 'package-install', - installType: 'update', - newVersion: '1.3.0', - packageName: 'apache', - status: 'success', + it('should do nothing if same version is installed', async () => { + jest.mocked(getInstallationObject).mockResolvedValueOnce({ + attributes: { + version: '1.2.0', + install_status: 'installed', + installed_es: [], + installed_kibana: [], + }, + } as any); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.2.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('already_installed'); }); - }); - it('should send telemetry on install failure, async error', async () => { - jest.mocked(install._installPackage).mockRejectedValue(new Error('error')); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => { + jest.mocked(appContextService.getConfig).mockReturnValueOnce({ + internal: { + fleetServerStandalone: true, + }, + } as any); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'fleet_server-2.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('installed'); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - errorMessage: 'error', - eventType: 'package-install', - installType: 'install', - newVersion: '1.3.0', - packageName: 'apache', - status: 'failure', + it('should use a scoped to package space soClient for tagging', async () => { + const mockedTaggingSo = savedObjectsClientMock.create(); + jest + .mocked(appContextService.getInternalUserSOClientForSpaceId) + .mockReturnValue(mockedTaggingSo); + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: 'test', + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(appContextService.getInternalUserSOClientForSpaceId).toBeCalledWith('test'); + expect(appContextService.getSavedObjectsTagging().createTagClient).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); + expect( + appContextService.getSavedObjectsTagging().createInternalAssignmentService + ).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); }); }); - it('should install from bundled package if one exists', async () => { - (install._installPackage as jest.Mock).mockResolvedValue({}); - mockGetBundledPackageByPkgKey.mockResolvedValue({ - name: 'test_package', - version: '1.0.0', - getBuffer: async () => Buffer.from('test_package'), + describe('with enablePackagesStateMachine = true', () => { + beforeEach(() => { + mockGetBundledPackageByPkgKey.mockResolvedValue(undefined); + jest.mocked(appContextService.getExperimentalFeatures).mockReturnValue({ + enablePackagesStateMachine: true, + } as any); }); - - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'test_package-1.0.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + afterEach(() => { + (install._installPackage as jest.Mock).mockClear(); + // jest.resetAllMocks(); + }); + afterAll(() => { + jest.mocked(appContextService.getExperimentalFeatures).mockReturnValue({ + enablePackagesStateMachine: false, + } as any); }); - expect(response.error).toBeUndefined(); - - expect(install._installPackage).toHaveBeenCalledWith( - expect.objectContaining({ installSource: 'bundled' }) - ); - }); + it('should send telemetry on install failure, out of date', async () => { + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.1.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'apache-1.1.0 is out-of-date and cannot be installed or updated', + eventType: 'package-install', + installType: 'install', + newVersion: '1.1.0', + packageName: 'apache', + status: 'failure', + }); + }); - it('should fetch latest version if version not provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'test_package', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on install failure, license error', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'Installation requires basic license', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - expect(response.status).toEqual('installed'); + it('should send telemetry on install success', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'success', + }); + }); - expect(sendTelemetryEvents).toHaveBeenCalledWith( - expect.anything(), - undefined, - expect.objectContaining({ + it('should send telemetry on update success', async () => { + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0', installed_kibana: [] } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: '1.2.0', + dryRun: false, + eventType: 'package-install', + installType: 'update', newVersion: '1.3.0', - }) - ); - }); + packageName: 'apache', + status: 'success', + }); + }); - it('should do nothing if same version is installed', async () => { - jest.mocked(getInstallationObject).mockResolvedValueOnce({ - attributes: { - version: '1.2.0', - install_status: 'installed', - installed_es: [], - installed_kibana: [], - }, - } as any); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.2.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on install failure, async error', async () => { + jest + .mocked(installStateMachine._stateMachineInstallPackage) + .mockRejectedValue(new Error('error')); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'error', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - expect(response.status).toEqual('already_installed'); - }); + it('should install from bundled package if one exists', async () => { + (installStateMachine._stateMachineInstallPackage as jest.Mock).mockResolvedValue({}); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + mockGetBundledPackageByPkgKey.mockResolvedValue({ + name: 'test_package', + version: '1.0.0', + getBuffer: async () => Buffer.from('test_package'), + }); + + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package-1.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.error).toBeUndefined(); + + expect(install._installPackage).toHaveBeenCalledWith( + expect.objectContaining({ installSource: 'bundled' }) + ); + }); - it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => { - jest.mocked(appContextService.getConfig).mockReturnValueOnce({ - internal: { - fleetServerStandalone: true, - }, - } as any); + it('should fetch latest version if version not provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('installed'); + + expect(sendTelemetryEvents).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.objectContaining({ + newVersion: '1.3.0', + }) + ); + }); - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'fleet_server-2.0.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should do nothing if same version is installed', async () => { + jest.mocked(getInstallationObject).mockResolvedValueOnce({ + attributes: { + version: '1.2.0', + install_status: 'installed', + installed_es: [], + installed_kibana: [], + }, + } as any); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.2.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('already_installed'); }); - expect(response.status).toEqual('installed'); - }); + // failing + it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => { + jest.mocked(appContextService.getConfig).mockReturnValueOnce({ + internal: { + fleetServerStandalone: true, + }, + } as any); - it('should use a scopped to package space soClient for tagging', async () => { - const mockedTaggingSo = savedObjectsClientMock.create(); - jest - .mocked(appContextService.getInternalUserSOClientForSpaceId) - .mockReturnValue(mockedTaggingSo); - jest - .mocked(getInstallationObject) - .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'fleet_server-2.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - await installPackage({ - spaceId: 'test', - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + expect(response.status).toEqual('installed'); }); - expect(appContextService.getInternalUserSOClientForSpaceId).toBeCalledWith('test'); - expect(appContextService.getSavedObjectsTagging().createTagClient).toBeCalledWith( - expect.objectContaining({ - client: mockedTaggingSo, - }) - ); - expect( - appContextService.getSavedObjectsTagging().createInternalAssignmentService - ).toBeCalledWith( - expect.objectContaining({ - client: mockedTaggingSo, - }) - ); + it('should use a scoped to package space soClient for tagging', async () => { + const mockedTaggingSo = savedObjectsClientMock.create(); + jest + .mocked(appContextService.getInternalUserSOClientForSpaceId) + .mockReturnValue(mockedTaggingSo); + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0', installed_kibana: [] } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: 'test', + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(appContextService.getInternalUserSOClientForSpaceId).toBeCalledWith('test'); + expect(appContextService.getSavedObjectsTagging().createTagClient).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); + expect( + appContextService.getSavedObjectsTagging().createInternalAssignmentService + ).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); + }); }); }); @@ -453,6 +718,7 @@ describe('install', () => { it('should send telemetry on install failure, async error', async () => { jest.mocked(install._installPackage).mockRejectedValue(new Error('error')); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); await installPackage({ spaceId: DEFAULT_SPACE_ID, installSource: 'upload', From 16ac037401eedeb363b6beea32e27fbc0b8926ca Mon Sep 17 00:00:00 2001 From: criamico Date: Wed, 27 Mar 2024 15:28:14 +0100 Subject: [PATCH 16/38] Save latest executed state in saved object and improve unit tests --- .../plugins/fleet/common/types/models/epm.ts | 30 ++ .../fleet/server/routes/epm/handlers.ts | 1 + .../fleet/server/saved_objects/index.ts | 11 + .../services/epm/kibana/assets/install.ts | 2 - .../services/epm/packages/_install_package.ts | 1 - .../_state_machine_package_install.ts | 33 +-- .../services/epm/packages/install_steps.ts | 28 +- .../integrations_state_machine.test.ts | 276 +++++++++++++----- .../packages/integrations_state_machine.ts | 21 +- x-pack/plugins/fleet/server/types/index.tsx | 1 + 10 files changed, 309 insertions(+), 95 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index cca3ede2595ac..a606e618b8731 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -549,6 +549,35 @@ export interface InstallFailedAttempt { }; } +const installStateNames = [ + 'create_restart_installation', + 'install_kibana_assets', + 'install_ilm_policies', + 'install_ml_model', + 'install_index_template_pipelines', + 'remove_legacy_templates', + 'update_current_write_indices', + 'install_transforms', + 'delete_previous_pipelines', + 'save_archive_entries_from_assets_map', + 'update_so', +] as const; + +type StateNamesTuple = typeof installStateNames; +export type StateNames = StateNamesTuple[number]; +export interface LatestExecutedState { + name: T; + started_at: string; + error?: string; +} + +export type InstallLatestExecutedState = LatestExecutedState; + +export interface StateContext { + [key: string]: any; + latestExecutedState: LatestExecutedState; +} + export interface Installation { installed_kibana: KibanaAssetReference[]; installed_es: EsAssetReference[]; @@ -569,6 +598,7 @@ export interface Installation { internal?: boolean; removable?: boolean; latest_install_failed_attempts?: InstallFailedAttempt[]; + latest_executed_state?: InstallLatestExecutedState; } export interface PackageUsageStats { diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 5c14cf3d3bac8..47324cfc493f1 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -612,6 +612,7 @@ const soToInstallationInfo = (pkg: PackageListItem | PackageInfo) => { verification_key_id: attributes.verification_key_id, experimental_data_stream_features: attributes.experimental_data_stream_features, latest_install_failed_attempts: attributes.latest_install_failed_attempts, + latest_executed_state: attributes.latest_executed_state, }; return { diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 0e44db492b2c1..4a441dc3fea0a 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -521,6 +521,7 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, }, latest_install_failed_attempts: { type: 'object', enabled: false }, + latest_executed_state: { type: 'object', enabled: false }, installed_kibana: { dynamic: false, properties: {}, @@ -562,6 +563,16 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, ], }, + '2': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + latest_executed_state: { type: 'object', enabled: false }, + }, + }, + ], + }, }, migrations: { '7.14.0': migrateInstallationToV7140, diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts index 47c4da20b9d05..2956cb5fe20c2 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts @@ -172,7 +172,6 @@ export async function installKibanaAssetsAndReferences({ pkgName, pkgTitle, packageInstallContext, - paths, installedPkg, spaceId, assetTags, @@ -185,7 +184,6 @@ export async function installKibanaAssetsAndReferences({ pkgName: string; pkgTitle: string; packageInstallContext: PackageInstallContext; - paths: string[]; installedPkg?: SavedObject; spaceId: string; assetTags?: PackageSpecTags[]; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 8b8a44b55e222..4a6cb0306a9cb 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -161,7 +161,6 @@ export async function _installPackage({ pkgName, pkgTitle, packageInstallContext, - paths, installedPkg, logger, spaceId, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts index c20b3576a868f..a6cddad056978 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts @@ -15,7 +15,7 @@ import type { import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server'; import type { HTTPAuthorizationHeader } from '../../../../common/http_authorization_header'; -import type { PackageInstallContext } from '../../../../common/types'; +import type { PackageInstallContext, StateNames, StateContext } from '../../../../common/types'; import type { PackageAssetReference } from '../../../types'; import type { @@ -41,28 +41,12 @@ import { stepDeletePreviousPipelines, stepSaveArchiveEntries, stepSaveSystemObject, + updateLatestExecutedState, } from './install_steps'; import type { StateMachineDefinition } from './integrations_state_machine'; import { handleState } from './integrations_state_machine'; -export const installStateNames = [ - 'create_restart_installation', - 'install_kibana_assets', - 'install_ilm_policies', - 'install_ml_model', - 'install_index_template_pipelines', - 'remove_legacy_templates', - 'update_current_write_indices', - 'install_transforms', - 'delete_previous_pipelines', - 'save_archive_entries_from_assets_map', - 'update_so', -] as const; - -type StateNamesTuple = typeof installStateNames; -type StateNames = StateNamesTuple[number]; - -export interface InstallContext { +export interface InstallContext extends StateContext { savedObjectsClient: SavedObjectsClientContract; savedObjectsImporter: Pick; savedObjectTagAssignmentService: IAssignmentService; @@ -96,46 +80,57 @@ export async function _stateMachineInstallPackage( create_restart_installation: { nextState: 'install_kibana_assets', onTransition: stepCreateRestartInstallation, + onPostTransition: updateLatestExecutedState, }, install_kibana_assets: { onTransition: stepInstallKibanaAssets, nextState: 'install_ilm_policies', + onPostTransition: updateLatestExecutedState, }, install_ilm_policies: { onTransition: stepInstallILMPolicies, nextState: 'install_ml_model', + onPostTransition: updateLatestExecutedState, }, install_ml_model: { onTransition: stepInstallMlModel, nextState: 'install_index_template_pipelines', + onPostTransition: updateLatestExecutedState, }, install_index_template_pipelines: { onTransition: stepInstallIndexTemplatePipelines, nextState: 'remove_legacy_templates', + onPostTransition: updateLatestExecutedState, }, remove_legacy_templates: { onTransition: stepRemoveLegacyTemplates, nextState: 'update_current_write_indices', + onPostTransition: updateLatestExecutedState, }, update_current_write_indices: { onTransition: stepUpdateCurrentWriteIndices, nextState: 'install_transforms', + onPostTransition: updateLatestExecutedState, }, install_transforms: { onTransition: stepInstallTransforms, nextState: 'delete_previous_pipelines', + onPostTransition: updateLatestExecutedState, }, delete_previous_pipelines: { onTransition: stepDeletePreviousPipelines, nextState: 'save_archive_entries_from_assets_map', + onPostTransition: updateLatestExecutedState, }, save_archive_entries_from_assets_map: { onTransition: stepSaveArchiveEntries, nextState: 'update_so', + onPostTransition: updateLatestExecutedState, }, update_so: { onTransition: stepSaveSystemObject, nextState: 'end', + onPostTransition: updateLatestExecutedState, }, }, }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts index 6d74474d4495b..ce775b8cec03b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; + import { ConcurrentInstallOperationError } from '../../../errors'; import { MAX_TIME_COMPLETE_INSTALL, @@ -112,7 +114,6 @@ export async function stepCreateRestartInstallation(context: InstallContext) { verificationResult, }); } - // Use a shared array that is updated by each operation. This allows each operation to accurately update the // installation object with it's references without requiring a refresh of the SO index on each update (faster). const esReferences = installedPkg?.attributes.installed_es ?? []; @@ -130,7 +131,7 @@ export async function stepInstallKibanaAssets(context: InstallContext) { packageInstallContext, spaceId, } = context; - const { packageInfo, paths } = packageInstallContext; + const { packageInfo } = packageInstallContext; const { name: pkgName, title: pkgTitle } = packageInfo; const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => @@ -142,7 +143,6 @@ export async function stepInstallKibanaAssets(context: InstallContext) { pkgName, pkgTitle, packageInstallContext, - paths, installedPkg, logger, spaceId, @@ -427,3 +427,25 @@ export async function stepSaveSystemObject(context: InstallContext) { `Install status ${updatedPackage?.attributes?.install_status} - Installation complete!` ); } + +// Function invoked after each transition +export const updateLatestExecutedState = async (context: InstallContext) => { + const { logger, savedObjectsClient, packageInstallContext, latestExecutedState } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName } = packageInfo; + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + try { + return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + latest_executed_state: latestExecutedState, + }); + } catch (err) { + if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { + logger.error(`failed to update package install state to: latest_executed_state ${err}`); + } + } +}; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index 40044ed53d674..ccf1f0b641029 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -92,26 +92,6 @@ describe('handleState', () => { expect(mockOnTransitionState3).toHaveBeenCalledTimes(1); }); - it('should exit when a state returns error', async () => { - const error = new Error('Installation failed'); - const mockOnTransitionState1 = jest.fn().mockRejectedValue(error); - const mockOnTransitionState2 = jest.fn(); - const mockOnTransitionState3 = jest.fn(); - const testDefinition = getTestDefinition( - mockOnTransitionState1, - mockOnTransitionState2, - mockOnTransitionState3 - ); - await handleState('state1', testDefinition, testDefinition.context); - - expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); - expect(mockOnTransitionState2).toHaveBeenCalledTimes(0); - expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); - expect(mockContract.logger?.warn).toHaveBeenCalledWith( - 'Error during execution of state "state1" with status "failed": Installation failed' - ); - }); - it('should call the onTransition function with context data and the return value is saved for the next iteration', async () => { const mockOnTransitionState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); const mockOnTransitionState2 = jest @@ -128,15 +108,27 @@ describe('handleState', () => { await handleState('state1', testDefinition, testDefinition.context); expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockOnTransitionState2).toHaveBeenCalledWith({ - testData: 'test', - arrayData: ['test1', 'test2'], - }); - expect(mockOnTransitionState3).toHaveBeenCalledWith({ - testData: 'test', - arrayData: ['test1', 'test2'], - promiseData: {}, - }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + promiseData: {}, + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); expect(mockContract.logger?.debug).toHaveBeenCalledWith( 'Executed state: state1 with status: success - nextState: state2' @@ -153,7 +145,7 @@ describe('handleState', () => { const mockOnTransitionState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); const state2Result = () => { return { - innerData: 'test', + result: 'test', }; }; const mockOnTransitionState2 = jest.fn().mockImplementation(() => { @@ -170,16 +162,28 @@ describe('handleState', () => { await handleState('state1', testDefinition, testDefinition.context); expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockOnTransitionState2).toHaveBeenCalledWith({ - testData: 'test', - arrayData: ['test1', 'test2'], - state2Result, - }); - expect(mockOnTransitionState3).toHaveBeenCalledWith({ - testData: 'test', - arrayData: ['test1', 'test2'], - state2Result, - }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + state2Result, + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + state2Result, + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); expect(mockContract.logger?.debug).toHaveBeenCalledWith( 'Executed state: state1 with status: success - nextState: state2' @@ -192,13 +196,13 @@ describe('handleState', () => { ); }); - it('should return updated context data', async () => { + it('should return updated context data ', async () => { const mockOnTransitionState1 = jest .fn() .mockImplementation(() => Promise.resolve({ promiseData: {} })); const state2Result = () => { return { - innerData: 'test', + result: 'test', }; }; const mockOnTransitionState2 = jest.fn().mockImplementation(() => { @@ -215,23 +219,41 @@ describe('handleState', () => { const updatedContext = await handleState('state1', testDefinition, testDefinition.context); expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockOnTransitionState2).toHaveBeenCalledWith({ - testData: 'test', - promiseData: {}, - state2Result, - }); - expect(mockOnTransitionState3).toHaveBeenCalledWith({ - testData: 'test', - promiseData: {}, - state2Result, - }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + promiseData: {}, + state2Result, + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + promiseData: {}, + state2Result, + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); - expect(updatedContext).toEqual({ - testData: 'test', - promiseData: {}, - state2Result, - lastData: ['test3'], - }); + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + promiseData: {}, + state2Result, + lastData: ['test3'], + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); }); it('should update a variable in the context at every call and return the updated value', async () => { @@ -250,15 +272,97 @@ describe('handleState', () => { const updatedContext = await handleState('state1', testDefinition, testDefinition.context); expect(mockOnTransitionState1).toHaveBeenCalledWith({ runningVal: [], fixedVal: 'something' }); - expect(mockOnTransitionState2).toHaveBeenCalledWith({ - runningVal: 'test1', - fixedVal: 'something', - }); - expect(mockOnTransitionState3).toHaveBeenCalledWith({ - runningVal: 'test2', - fixedVal: 'something', - }); - expect(updatedContext).toEqual({ fixedVal: 'something', runningVal: 'test3' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: 'test1', + fixedVal: 'something', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: 'test2', + fixedVal: 'something', + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); + expect(updatedContext).toEqual( + expect.objectContaining({ + fixedVal: 'something', + runningVal: 'test3', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + }); + + it('should exit when a state returns error', async () => { + const error = new Error('Installation failed'); + const mockOnTransitionState1 = jest.fn().mockRejectedValue(error); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3 + ); + await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(0); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Error during execution of state "state1" with status "failed": Installation failed' + ); + }); + + it('should exit when a state returns error and keep info about latest executed step', async () => { + const error = new Error('Installation failed'); + const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); + const mockOnTransitionState2 = jest.fn().mockRejectedValue(error); + const mockOnTransitionState3 = jest.fn(); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + const result = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Error during execution of state "state2" with status "failed": Installation failed' + ); + expect(result).toEqual( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); }); it('should execute postTransition function after the transition is complete', async () => { @@ -285,4 +389,42 @@ describe('handleState', () => { 'Executing post transition function: mockConstructor' ); }); + + it('should execute postTransition function after the transition passing the updated context', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ runningVal: 'test1' }); + const mockOnTransitionState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ runningVal: 'test2' })); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ runningVal: 'test3' }); + const mockPostTransition = jest.fn(); + const contextData = { fixedVal: 'something' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData, + mockPostTransition + ); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockOnTransitionState2).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockOnTransitionState3).toHaveBeenCalled(); + expect(updatedContext).toEqual( + expect.objectContaining({ + fixedVal: 'something', + runningVal: 'test3', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + expect(mockPostTransition).toHaveBeenCalledWith(updatedContext); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executing post transition function: mockConstructor' + ); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index dc68cc2886dbe..d3cf65dfd1157 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -5,6 +5,7 @@ * 2.0. */ import { appContextService } from '../../app_context'; +import type { StateContext } from '../../../../common/types'; export interface State { onTransition: any; nextState?: string; @@ -22,8 +23,8 @@ export interface StateMachineDefinition { export async function handleState( currentStateName: string, definition: StateMachineDefinition, - context: { [key: string]: any } -): Promise<{ [key: string]: any }> { + context: StateContext +): Promise> { const logger = appContextService.getLogger(); const { states } = definition; const currentState = states[currentStateName]; @@ -35,13 +36,24 @@ export async function handleState( `Current state ${currentStateName}: running transition ${currentState.onTransition.name}` ); try { + // inject information about the state into context + const startedAt = new Date(Date.now()).toISOString(); + const latestExecutedState = { + name: currentStateName, + started_at: startedAt, + }; stateResult = await currentState.onTransition.call(undefined, updatedContext); // check if is a function/promise if (typeof stateResult === 'function') { const promiseName = `${currentStateName}Result`; updatedContext[promiseName] = stateResult; + updatedContext = { ...updatedContext, currentStateName, latestExecutedState }; } else { - updatedContext = { ...updatedContext, ...stateResult }; + updatedContext = { + ...updatedContext, + ...stateResult, + latestExecutedState, + }; } currentStatus = 'success'; logger.debug( @@ -55,6 +67,9 @@ export async function handleState( } } else { currentStatus = 'failed'; + logger.warn( + `Execution of state "${currentStateName}" with status "${currentStatus}": provided onTransition is not a valid function ` + ); } if (typeof currentState.onPostTransition === 'function') { diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 18a3ed05bfcd2..da4d793989e8b 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -98,6 +98,7 @@ export type { PackageSpecTags, AssetsMap, InstallResultStatus, + InstallLatestExecutedState, } from '../../common/types'; export { ElasticsearchAssetType, KibanaAssetType, KibanaSavedObjectType } from '../../common/types'; export { dataTypes } from '../../common/constants'; From a82bd682e28041e255bb8270d3beed4b61bb3035 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:20:38 +0000 Subject: [PATCH 17/38] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- packages/kbn-check-mappings-update-cli/current_fields.json | 1 + packages/kbn-check-mappings-update-cli/current_mappings.json | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 11a0173c973ce..63c2902d329b6 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -290,6 +290,7 @@ "installed_kibana_space_id", "internal", "keep_policies_up_to_date", + "latest_executed_state", "latest_install_failed_attempts", "name", "package_assets", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 2096be33bff9e..c8bde85fb1ac1 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1003,6 +1003,10 @@ "index": false, "type": "boolean" }, + "latest_executed_state": { + "enabled": false, + "type": "object" + }, "latest_install_failed_attempts": { "enabled": false, "type": "object" From cefff9b151776cba4ee6790f5e9f1d376d1a850a Mon Sep 17 00:00:00 2001 From: criamico Date: Wed, 27 Mar 2024 17:33:21 +0100 Subject: [PATCH 18/38] Save error in saved object --- .../epm/packages/integrations_state_machine.test.ts | 2 ++ .../epm/packages/integrations_state_machine.ts | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index ccf1f0b641029..9d2e10a28cb24 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -360,6 +360,8 @@ describe('handleState', () => { latestExecutedState: { name: 'state1', started_at: expect.anything(), + errors: + 'Error during execution of state "state2" with status "failed": Installation failed', }, }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index d3cf65dfd1157..cb61e79d2c69b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -47,7 +47,7 @@ export async function handleState( if (typeof stateResult === 'function') { const promiseName = `${currentStateName}Result`; updatedContext[promiseName] = stateResult; - updatedContext = { ...updatedContext, currentStateName, latestExecutedState }; + updatedContext = { ...updatedContext, latestExecutedState }; } else { updatedContext = { ...updatedContext, @@ -61,14 +61,15 @@ export async function handleState( ); } catch (error) { currentStatus = 'failed'; - logger.warn( - `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}` - ); + const errorMessage = `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}`; + logger.warn(errorMessage); + const latestStateWithError = { ...updatedContext.latestExecutedState, errors: errorMessage }; + updatedContext = { ...updatedContext, latestExecutedState: latestStateWithError }; } } else { currentStatus = 'failed'; logger.warn( - `Execution of state "${currentStateName}" with status "${currentStatus}": provided onTransition is not a valid function ` + `Execution of state "${currentStateName}" with status "${currentStatus}": provided onTransition is not a valid function` ); } From fa85aa69256c356700d4e0e1ba21d2fbde1ff7ed Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 28 Mar 2024 12:00:43 +0100 Subject: [PATCH 19/38] Add unit tests --- .../integrations_state_machine.test.ts | 200 ++++++++++++++++-- .../packages/integrations_state_machine.ts | 2 +- 2 files changed, 182 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index 9d2e10a28cb24..d58807dcca264 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -76,22 +76,6 @@ describe('handleState', () => { ); }); - it('should execute the transition from the provided state', async () => { - const mockOnTransitionState1 = jest.fn(); - const mockOnTransitionState2 = jest.fn(); - const mockOnTransitionState3 = jest.fn(); - const testDefinition = getTestDefinition( - mockOnTransitionState1, - mockOnTransitionState2, - mockOnTransitionState3 - ); - await handleState('state2', testDefinition, testDefinition.context); - - expect(mockOnTransitionState1).toHaveBeenCalledTimes(0); - expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); - expect(mockOnTransitionState3).toHaveBeenCalledTimes(1); - }); - it('should call the onTransition function with context data and the return value is saved for the next iteration', async () => { const mockOnTransitionState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); const mockOnTransitionState2 = jest @@ -196,7 +180,7 @@ describe('handleState', () => { ); }); - it('should return updated context data ', async () => { + it('should return updated context data', async () => { const mockOnTransitionState1 = jest .fn() .mockImplementation(() => Promise.resolve({ promiseData: {} })); @@ -304,6 +288,51 @@ describe('handleState', () => { ); }); + it('should execute the transition starting from the provided state', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ runningVal: 'test1' }); + const mockOnTransitionState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ runningVal: 'test2' })); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ runningVal: 'test3' }); + const contextData = { runningVal: [], fixedVal: 'something' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + + const updatedContext = await handleState('state2', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(0); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: [], + fixedVal: 'something', + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: 'test2', + fixedVal: 'something', + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); + expect(updatedContext).toEqual( + expect.objectContaining({ + fixedVal: 'something', + runningVal: 'test3', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + }); + it('should exit when a state returns error', async () => { const error = new Error('Installation failed'); const mockOnTransitionState1 = jest.fn().mockRejectedValue(error); @@ -336,7 +365,7 @@ describe('handleState', () => { mockOnTransitionState3, contextData ); - const result = await handleState('state1', testDefinition, testDefinition.context); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); expect(mockOnTransitionState2).toHaveBeenCalledWith( @@ -353,7 +382,7 @@ describe('handleState', () => { expect(mockContract.logger?.warn).toHaveBeenCalledWith( 'Error during execution of state "state2" with status "failed": Installation failed' ); - expect(result).toEqual( + expect(updatedContext).toEqual( expect.objectContaining({ testData: 'test', result1: 'test', @@ -429,4 +458,137 @@ describe('handleState', () => { 'Executing post transition function: mockConstructor' ); }); + + it('should execute postTransition correctly also when a transition exits with erros', async () => { + const error = new Error('Installation failed'); + const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); + const mockOnTransitionState2 = jest.fn().mockRejectedValue(error); + const mockOnTransitionState3 = jest.fn(); + const mockPostTransition = jest.fn(); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData, + mockPostTransition + ); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockPostTransition).toHaveBeenCalledWith( + expect.objectContaining({ + result1: 'test', + testData: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); + expect(mockPostTransition).toHaveBeenCalledWith( + expect.objectContaining({ + result1: 'test', + testData: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + errors: + 'Error during execution of state "state2" with status "failed": Installation failed', + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); + + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + errors: + 'Error during execution of state "state2" with status "failed": Installation failed', + }, + }) + ); + }); + + it('should log a warning when postTransition exits with erros and continue executing the states', async () => { + const error = new Error('Installation failed'); + const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const mockPostTransition = jest.fn().mockRejectedValue(error); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData, + mockPostTransition + ); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockPostTransition).toHaveBeenCalledWith( + expect.objectContaining({ + result1: 'test', + testData: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(1); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Error during execution of post transition function: Installation failed' + ); + + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + }); + + it('should exit and log a warning when the provided OnTransition is not a function', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); + const mockOnTransitionState2 = undefined; + const mockOnTransitionState3 = jest.fn(); + + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Execution of state "state2" with status "failed": provided onTransition is not a valid function' + ); + + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index cb61e79d2c69b..0d2f9f284beb5 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -62,9 +62,9 @@ export async function handleState( } catch (error) { currentStatus = 'failed'; const errorMessage = `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}`; - logger.warn(errorMessage); const latestStateWithError = { ...updatedContext.latestExecutedState, errors: errorMessage }; updatedContext = { ...updatedContext, latestExecutedState: latestStateWithError }; + logger.warn(errorMessage); } } else { currentStatus = 'failed'; From 18ded6f6f14aefbf61c288e79294c698323e5abb Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 28 Mar 2024 12:19:31 +0100 Subject: [PATCH 20/38] Add openapi changes and fix wrong parameter name --- .../plugins/fleet/common/openapi/bundled.json | 28 +++++++++++++++++++ .../plugins/fleet/common/openapi/bundled.yaml | 22 +++++++++++++++ .../components/schemas/installation_info.yaml | 22 +++++++++++++++ .../integrations_state_machine.test.ts | 6 ++-- .../packages/integrations_state_machine.ts | 2 +- 5 files changed, 76 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index f7c192db0fe5e..bebac0c08da94 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -6263,6 +6263,34 @@ } } }, + "latest_executed_state": { + "description": "Latest successfully executed state in package install state machine", + "type": "object", + "properties": { + "name": { + "type": "string", + "enum": [ + "create_restart_installation", + "install_kibana_assets", + "install_ilm_policies", + "install_ml_model", + "install_index_template_pipelines", + "remove_legacy_templates", + "update_current_write_indices", + "install_transforms", + "delete_previous_pipelines", + "save_archive_entries_from_assets_map", + "update_so" + ] + }, + "started_at": { + "type": "string" + }, + "error": { + "type": "string" + } + } + }, "verification_status": { "type": "string", "enum": [ diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 406a2550d770d..345d04cf96c6d 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -3949,6 +3949,28 @@ components: type: string stack: type: string + latest_executed_state: + description: Latest successfully executed state in package install state machine + type: object + properties: + name: + type: string + enum: + - create_restart_installation + - install_kibana_assets + - install_ilm_policies + - install_ml_model + - install_index_template_pipelines + - remove_legacy_templates + - update_current_write_indices + - install_transforms + - delete_previous_pipelines + - save_archive_entries_from_assets_map + - update_so + started_at: + type: string + error: + type: string verification_status: type: string enum: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml index c5db5f12d4cc3..b8d82bc669d04 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml @@ -66,6 +66,28 @@ properties: type: string stack: type: string + latest_executed_state: + description: Latest successfully executed state in package install state machine + type: object + properties: + name: + type: string + enum: + - create_restart_installation + - install_kibana_assets + - install_ilm_policies + - install_ml_model + - install_index_template_pipelines + - remove_legacy_templates + - update_current_write_indices + - install_transforms + - delete_previous_pipelines + - save_archive_entries_from_assets_map + - update_so + started_at: + type: string + error: + type: string verification_status: type: string enum: diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index d58807dcca264..e83b234f3a520 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -389,7 +389,7 @@ describe('handleState', () => { latestExecutedState: { name: 'state1', started_at: expect.anything(), - errors: + error: 'Error during execution of state "state2" with status "failed": Installation failed', }, }) @@ -494,7 +494,7 @@ describe('handleState', () => { latestExecutedState: { name: 'state1', started_at: expect.anything(), - errors: + error: 'Error during execution of state "state2" with status "failed": Installation failed', }, }) @@ -508,7 +508,7 @@ describe('handleState', () => { latestExecutedState: { name: 'state1', started_at: expect.anything(), - errors: + error: 'Error during execution of state "state2" with status "failed": Installation failed', }, }) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 0d2f9f284beb5..5804fe3cedeb3 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -62,7 +62,7 @@ export async function handleState( } catch (error) { currentStatus = 'failed'; const errorMessage = `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}`; - const latestStateWithError = { ...updatedContext.latestExecutedState, errors: errorMessage }; + const latestStateWithError = { ...updatedContext.latestExecutedState, error: errorMessage }; updatedContext = { ...updatedContext, latestExecutedState: latestStateWithError }; logger.warn(errorMessage); } From 12a0e331daeb2b3143f3b3ff692ed31783b726c8 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 28 Mar 2024 11:48:34 +0000 Subject: [PATCH 21/38] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index fbc0d5b2ef269..03be764c40a87 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -85,7 +85,7 @@ describe('checking migration metadata changes on all registered SO types', () => "dashboard": "211e9ca30f5a95d5f3c27b1bf2b58e6cfa0c9ae9", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", - "epm-packages": "c23d3d00c051a08817335dba26f542b64b18a56a", + "epm-packages": "f8ee125b57df31fd035dc04ad81aef475fd2f5bd", "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1", "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582", "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88", From 6059c93c35e40da688202cf0ced584262ecf8100 Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 28 Mar 2024 13:02:13 +0100 Subject: [PATCH 22/38] Fix type error --- .../plugins/fleet/common/types/models/epm.ts | 2 +- .../_state_machine_package_install.test.ts | 439 ++++++++++++++++++ 2 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index a606e618b8731..87435c03eb468 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -575,7 +575,7 @@ export type InstallLatestExecutedState = LatestExecutedState; export interface StateContext { [key: string]: any; - latestExecutedState: LatestExecutedState; + latestExecutedState?: LatestExecutedState; } export interface Installation { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts new file mode 100644 index 0000000000000..44bea820272f2 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts @@ -0,0 +1,439 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { ConcurrentInstallOperationError, PackageSavedObjectConflictError } from '../../../errors'; + +import type { Installation } from '../../../../common'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; + +import { appContextService } from '../../app_context'; +import { createAppContextStartContractMock } from '../../../mocks'; +import { saveArchiveEntriesFromAssetsMap } from '../archive/storage'; +import { installILMPolicy } from '../elasticsearch/ilm/install'; +import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; + +jest.mock('../elasticsearch/template/template'); +jest.mock('../kibana/assets/install'); +jest.mock('../kibana/index_pattern/install'); +jest.mock('./install'); +jest.mock('./get'); +jest.mock('./install_index_template_pipeline'); + +jest.mock('../archive/storage'); +jest.mock('../elasticsearch/ilm/install'); +jest.mock('../elasticsearch/datastream_ilm/install'); + +import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; +import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; + +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../common/constants'; + +import { restartInstallation } from './install'; +import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; + +import { _stateMachineInstallPackage } from './_state_machine_package_install'; + +const mockedInstallIndexTemplatesAndPipelines = + installIndexTemplatesAndPipelines as jest.MockedFunction< + typeof installIndexTemplatesAndPipelines + >; +const mockedUpdateCurrentWriteIndices = updateCurrentWriteIndices as jest.MockedFunction< + typeof updateCurrentWriteIndices +>; +const mockedInstallKibanaAssetsAndReferences = + installKibanaAssetsAndReferences as jest.MockedFunction; + +function sleep(millis: number) { + return new Promise((resolve) => setTimeout(resolve, millis)); +} + +describe('_installPackage', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + + soClient.update.mockImplementation(async (type, id, attributes) => { + return { id, attributes } as any; + }); + soClient.get.mockImplementation(async (type, id) => { + return { id, attributes: {} } as any; + }); + + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + jest.mocked(installILMPolicy).mockReset(); + jest.mocked(installIlmForDataStream).mockReset(); + jest.mocked(installIlmForDataStream).mockResolvedValue({ + esReferences: [], + installedIlms: [], + }); + jest.mocked(saveArchiveEntriesFromAssetsMap).mockResolvedValue({ + saved_objects: [], + }); + jest.mocked(restartInstallation).mockReset(); + }); + it('handles errors from installKibanaAssets', async () => { + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockImplementation(async () => { + throw new Error('mocked async error A: should be caught'); + }); + + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + const installationPromise = _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + // if we have a .catch this will fail nicely (test pass) + // otherwise the test will fail with either of the mocked errors + await expect(installationPromise).rejects.toThrow('mocked'); + await expect(installationPromise).rejects.toThrow('should be caught'); + }); + + it('do not install ILM policies if disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]); + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).not.toBeCalled(); + expect(installIlmForDataStream).not.toBeCalled(); + // if we have a .catch this will fail nicely (test pass) + // otherwise the test will fail with either of the mocked errors + // await expect(installationPromise).rejects.toThrow('mocked'); + // await expect(installationPromise).rejects.toThrow('should be caught'); + }); + + it('install ILM policies if not disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]); + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).toBeCalled(); + expect(installIlmForDataStream).toBeCalled(); + }); + + describe('when package is stuck in `installing`', () => { + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(() => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + }); + + describe('timeout reached', () => { + it('restarts installation', async () => { + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + paths: [], + assetsMap: new Map(), + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date( + Date.now() - MAX_TIME_COMPLETE_INSTALL * 2 + ).toISOString(), + }, + }, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + + describe('timeout not reached', () => { + describe('force flag not provided', () => { + it('throws concurrent installation error if force flag is not provided', async () => { + expect( + _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + paths: [], + assetsMap: new Map(), + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + }) + ).rejects.toThrowError(ConcurrentInstallOperationError); + }); + }); + + describe('force flag provided', () => { + it('restarts installation', async () => { + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + paths: [], + assetsMap: new Map(), + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + force: true, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + }); + }); + + it('surfaces saved object conflicts error', () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + + mockedInstallKibanaAssetsAndReferences.mockRejectedValueOnce( + new PackageSavedObjectConflictError('test') + ); + + expect( + _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }) + ).rejects.toThrowError(PackageSavedObjectConflictError); + }); +}); From 57466105b02cf647b5ed22b742e73fd4d943c8e2 Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 29 Mar 2024 12:02:54 +0100 Subject: [PATCH 23/38] Fixes to the logic, add tests --- .../_state_machine_package_install.test.ts | 113 ++++++++++-------- .../_state_machine_package_install.ts | 52 +++++--- .../services/epm/packages/install_steps.ts | 4 + .../integrations_state_machine.test.ts | 80 +++---------- .../packages/integrations_state_machine.ts | 12 +- 5 files changed, 129 insertions(+), 132 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts index 44bea820272f2..e7a839981c9b7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts @@ -14,7 +14,7 @@ import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/serv import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; -import { ConcurrentInstallOperationError, PackageSavedObjectConflictError } from '../../../errors'; +import { PackageSavedObjectConflictError } from '../../../errors'; import type { Installation } from '../../../../common'; @@ -61,7 +61,7 @@ function sleep(millis: number) { return new Promise((resolve) => setTimeout(resolve, millis)); } -describe('_installPackage', () => { +describe('_stateMachineInstallPackage', () => { let soClient: jest.Mocked; let esClient: jest.Mocked; @@ -88,7 +88,8 @@ describe('_installPackage', () => { }); jest.mocked(restartInstallation).mockReset(); }); - it('handles errors from installKibanaAssets', async () => { + + it('handles errors from installKibanaAssets', async () => { // force errors from this function mockedInstallKibanaAssetsAndReferences.mockImplementation(async () => { throw new Error('mocked async error A: should be caught'); @@ -102,37 +103,42 @@ describe('_installPackage', () => { installedTemplates: [], esReferences: [], }); - const installationPromise = _stateMachineInstallPackage({ - savedObjectsClient: soClient, - // @ts-ignore - savedObjectsImporter: jest.fn(), - esClient, - logger: loggerMock.create(), - packageInstallContext: { - assetsMap: new Map(), - paths: [], - packageInfo: { - title: 'title', - name: 'xyz', - version: '4.5.6', - description: 'test', - type: 'integration', - categories: ['cloud', 'custom'], - format_version: 'string', - release: 'experimental', - conditions: { kibana: { version: 'x.y.z' } }, - owner: { github: 'elastic/fleet' }, - }, - }, - installType: 'install', - installSource: 'registry', - spaceId: DEFAULT_SPACE_ID, - }); - // if we have a .catch this will fail nicely (test pass) - // otherwise the test will fail with either of the mocked errors - await expect(installationPromise).rejects.toThrow('mocked'); - await expect(installationPromise).rejects.toThrow('should be caught'); + // use this workaround to test the error; toThrow/toThrowError doesn't match correctly + let thrownError; + try { + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + } catch (error) { + thrownError = error; + } + expect(thrownError).toEqual( + `Error during execution of state \"save_archive_entries_from_assets_map\" with status \"failed\": mocked async error A: should be caught` + ); }); it('do not install ILM policies if disabled in config', async () => { @@ -190,10 +196,6 @@ describe('_installPackage', () => { expect(installILMPolicy).not.toBeCalled(); expect(installIlmForDataStream).not.toBeCalled(); - // if we have a .catch this will fail nicely (test pass) - // otherwise the test will fail with either of the mocked errors - // await expect(installationPromise).rejects.toThrow('mocked'); - // await expect(installationPromise).rejects.toThrow('should be caught'); }); it('install ILM policies if not disabled in config', async () => { @@ -322,11 +324,13 @@ describe('_installPackage', () => { }); }); - describe('timeout not reached', () => { - describe('force flag not provided', () => { + describe('Timeout not reached', () => { + describe('Force flag not provided', () => { it('throws concurrent installation error if force flag is not provided', async () => { - expect( - _stateMachineInstallPackage({ + // use this workaround to test the error; toThrow/toThrowError doesn't match correctly + let thrownError; + try { + await _stateMachineInstallPackage({ savedObjectsClient: soClient, // @ts-ignore savedObjectsImporter: jest.fn(), @@ -348,8 +352,13 @@ describe('_installPackage', () => { install_started_at: new Date(Date.now() - 1000).toISOString(), }, }, - }) - ).rejects.toThrowError(ConcurrentInstallOperationError); + }); + } catch (error) { + thrownError = error; + } + expect(thrownError).toEqual( + `Error during execution of state \"create_restart_installation\" with status \"failed\": Concurrent installation or upgrade of test-package-1.0.0 detected, aborting.` + ); }); }); @@ -386,7 +395,7 @@ describe('_installPackage', () => { }); }); - it('surfaces saved object conflicts error', () => { + it('surfaces saved object conflicts error', async () => { appContextService.start( createAppContextStartContractMock({ internal: { @@ -406,9 +415,10 @@ describe('_installPackage', () => { mockedInstallKibanaAssetsAndReferences.mockRejectedValueOnce( new PackageSavedObjectConflictError('test') ); - - expect( - _stateMachineInstallPackage({ + let thrownError; + // use this workaround to test the error; toThrow/toThrowError doesn't match correctly + try { + await _stateMachineInstallPackage({ savedObjectsClient: soClient, // @ts-ignore savedObjectsImporter: jest.fn(), @@ -433,7 +443,12 @@ describe('_installPackage', () => { installType: 'install', installSource: 'registry', spaceId: DEFAULT_SPACE_ID, - }) - ).rejects.toThrowError(PackageSavedObjectConflictError); + }); + } catch (error) { + thrownError = error; + } + expect(thrownError).toEqual( + `Error during execution of state "save_archive_entries_from_assets_map" with status "failed": test` + ); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts index a6cddad056978..b882775c7e500 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts @@ -11,9 +11,12 @@ import type { SavedObjectsClientContract, ISavedObjectsImporter, } from '@kbn/core/server'; +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server'; +import { PackageSavedObjectConflictError } from '../../../errors'; + import type { HTTPAuthorizationHeader } from '../../../../common/http_authorization_header'; import type { PackageInstallContext, StateNames, StateContext } from '../../../../common/types'; import type { PackageAssetReference } from '../../../types'; @@ -134,20 +137,37 @@ export async function _stateMachineInstallPackage( }, }, }; - const { installedKibanaAssetsRefs, esReferences } = await handleState( - 'create_restart_installation', - installStates, - installStates.context - ); - if ( - installedKibanaAssetsRefs && - installedKibanaAssetsRefs.length && - esReferences && - esReferences.length - ) - return [ - ...(installedKibanaAssetsRefs as KibanaAssetReference[]), - ...(esReferences as EsAssetReference[]), - ]; - return []; + try { + const { installedKibanaAssetsRefs, esReferences } = await handleState( + 'create_restart_installation', + installStates, + installStates.context + ); + if ( + installedKibanaAssetsRefs && + installedKibanaAssetsRefs.length && + esReferences && + esReferences.length + ) + return [ + ...(installedKibanaAssetsRefs as KibanaAssetReference[]), + ...(esReferences as EsAssetReference[]), + ]; + return []; + } catch (err) { + const { packageInfo } = installStates.context.packageInstallContext; + const { name: pkgName, version: pkgVersion } = packageInfo; + + if (SavedObjectsErrorHelpers.isConflictError(err)) { + throw new PackageSavedObjectConflictError( + `Saved Object conflict encountered while installing ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + }. There may be a conflicting Saved Object saved to another Space. Original error: ${ + err.message + }` + ); + } else { + throw err; + } + } } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts index ce775b8cec03b..b6e996ad5b1d3 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts @@ -149,6 +149,10 @@ export async function stepInstallKibanaAssets(context: InstallContext) { assetTags: packageInfo?.asset_tags, }) ); + // Necessary to avoid async promise rejection warning + // See https://stackoverflow.com/questions/40920179/should-i-refrain-from-handling-promise-rejection-asynchronously + kibanaAssetPromise.catch(() => {}); + return { kibanaAssetPromise }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts index e83b234f3a520..179ff7e01a97c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts @@ -333,7 +333,7 @@ describe('handleState', () => { ); }); - it('should exit when a state returns error', async () => { + it('should throw when a state returns error', async () => { const error = new Error('Installation failed'); const mockOnTransitionState1 = jest.fn().mockRejectedValue(error); const mockOnTransitionState2 = jest.fn(); @@ -343,8 +343,13 @@ describe('handleState', () => { mockOnTransitionState2, mockOnTransitionState3 ); - await handleState('state1', testDefinition, testDefinition.context); - + try { + await handleState('state1', testDefinition, testDefinition.context); + } catch (err) { + expect(err).toEqual( + `Error during execution of state "state1" with status "failed": Installation failed` + ); + } expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); expect(mockOnTransitionState2).toHaveBeenCalledTimes(0); expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); @@ -353,49 +358,6 @@ describe('handleState', () => { ); }); - it('should exit when a state returns error and keep info about latest executed step', async () => { - const error = new Error('Installation failed'); - const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); - const mockOnTransitionState2 = jest.fn().mockRejectedValue(error); - const mockOnTransitionState3 = jest.fn(); - const contextData = { testData: 'test' }; - const testDefinition = getTestDefinition( - mockOnTransitionState1, - mockOnTransitionState2, - mockOnTransitionState3, - contextData - ); - const updatedContext = await handleState('state1', testDefinition, testDefinition.context); - - expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); - expect(mockOnTransitionState2).toHaveBeenCalledWith( - expect.objectContaining({ - testData: 'test', - result1: 'test', - latestExecutedState: { - name: 'state1', - started_at: expect.anything(), - }, - }) - ); - expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); - expect(mockContract.logger?.warn).toHaveBeenCalledWith( - 'Error during execution of state "state2" with status "failed": Installation failed' - ); - expect(updatedContext).toEqual( - expect.objectContaining({ - testData: 'test', - result1: 'test', - latestExecutedState: { - name: 'state1', - started_at: expect.anything(), - error: - 'Error during execution of state "state2" with status "failed": Installation failed', - }, - }) - ); - }); - it('should execute postTransition function after the transition is complete', async () => { const mockOnTransitionState1 = jest.fn(); const mockOnTransitionState2 = jest.fn(); @@ -459,7 +421,7 @@ describe('handleState', () => { ); }); - it('should execute postTransition correctly also when a transition exits with erros', async () => { + it('should not execute postTransition when a transition exits with errors', async () => { const error = new Error('Installation failed'); const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); const mockOnTransitionState2 = jest.fn().mockRejectedValue(error); @@ -473,8 +435,13 @@ describe('handleState', () => { contextData, mockPostTransition ); - const updatedContext = await handleState('state1', testDefinition, testDefinition.context); - + try { + await handleState('state1', testDefinition, testDefinition.context); + } catch (err) { + expect(err).toEqual( + `Error during execution of state \"state2\" with status \"failed\": Installation failed` + ); + } expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); expect(mockPostTransition).toHaveBeenCalledWith( expect.objectContaining({ @@ -494,25 +461,10 @@ describe('handleState', () => { latestExecutedState: { name: 'state1', started_at: expect.anything(), - error: - 'Error during execution of state "state2" with status "failed": Installation failed', }, }) ); expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); - - expect(updatedContext).toEqual( - expect.objectContaining({ - testData: 'test', - result1: 'test', - latestExecutedState: { - name: 'state1', - started_at: expect.anything(), - error: - 'Error during execution of state "state2" with status "failed": Installation failed', - }, - }) - ); }); it('should log a warning when postTransition exits with erros and continue executing the states', async () => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts index 5804fe3cedeb3..7cc56a8d3f15e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts @@ -5,7 +5,7 @@ * 2.0. */ import { appContextService } from '../../app_context'; -import type { StateContext } from '../../../../common/types'; +import type { StateContext, LatestExecutedState } from '../../../../common/types'; export interface State { onTransition: any; nextState?: string; @@ -38,7 +38,7 @@ export async function handleState( try { // inject information about the state into context const startedAt = new Date(Date.now()).toISOString(); - const latestExecutedState = { + const latestExecutedState: LatestExecutedState = { name: currentStateName, started_at: startedAt, }; @@ -62,9 +62,15 @@ export async function handleState( } catch (error) { currentStatus = 'failed'; const errorMessage = `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}`; - const latestStateWithError = { ...updatedContext.latestExecutedState, error: errorMessage }; + const latestStateWithError = { + ...updatedContext.latestExecutedState, + error: errorMessage, + } as LatestExecutedState; updatedContext = { ...updatedContext, latestExecutedState: latestStateWithError }; logger.warn(errorMessage); + + // bubble up the error + throw errorMessage; } } else { currentStatus = 'failed'; From f88752dca1990b56bf80e1e15f39c20f8acfec4d Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 29 Mar 2024 12:37:05 +0100 Subject: [PATCH 24/38] Split up steps in smaller functions and improve files organization --- .../services/epm/packages/install.test.ts | 4 +- .../server/services/epm/packages/install.ts | 2 +- .../_state_machine_package_install.test.ts | 44 +- .../_state_machine_package_install.ts | 16 +- .../state_machine.test.ts} | 6 +- .../state_machine.ts} | 4 +- .../install_state_machine/steps/index.ts | 19 + .../steps/step_create_restart_installation.ts | 87 ++++ .../steps/step_delete_previous_pipelines.ts | 63 +++ .../steps/step_install_ML_model.ts | 21 + .../steps/step_install_ilm_policies.ts | 50 ++ .../step_install_index_template_pipelines.ts | 66 +++ .../steps/step_install_kibana_assets.ts | 48 ++ .../steps/step_install_transforms.ts | 38 ++ .../steps/step_remove_legacy_templates.ts | 20 + .../steps/step_save_archive_entries.ts | 40 ++ .../steps/step_save_system_object.ts | 82 ++++ .../step_update_current_write_indices.ts | 25 + .../steps/update_latest_executed_state.ts | 36 ++ .../services/epm/packages/install_steps.ts | 455 ------------------ 20 files changed, 633 insertions(+), 493 deletions(-) rename x-pack/plugins/fleet/server/services/epm/packages/{ => install_state_machine}/_state_machine_package_install.test.ts (91%) rename x-pack/plugins/fleet/server/services/epm/packages/{ => install_state_machine}/_state_machine_package_install.ts (92%) rename x-pack/plugins/fleet/server/services/epm/packages/{integrations_state_machine.test.ts => install_state_machine/state_machine.test.ts} (99%) rename x-pack/plugins/fleet/server/services/epm/packages/{integrations_state_machine.ts => install_state_machine/state_machine.ts} (97%) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ML_model.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts delete mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index 921dbe63809cb..bbaa10728754b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -29,7 +29,7 @@ import { isPackageVersionOrLaterInstalled, } from './install'; import * as install from './_install_package'; -import * as installStateMachine from './_state_machine_package_install'; +import * as installStateMachine from './install_state_machine/_state_machine_package_install'; import { getBundledPackageByPkgKey } from './bundled_packages'; import { getInstalledPackageWithAssets, getInstallationObject } from './get'; @@ -81,7 +81,7 @@ jest.mock('./_install_package', () => { _installPackage: jest.fn(() => Promise.resolve()), }; }); -jest.mock('./_state_machine_package_install', () => { +jest.mock('./install_state_machine/_state_machine_package_install', () => { return { _stateMachineInstallPackage: jest.fn(() => Promise.resolve()), }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index b3419469ebcc5..c8c2e542bfe8b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -71,7 +71,7 @@ import { sendTelemetryEvents, UpdateEventType } from '../../upgrade_sender'; import { auditLoggingService } from '../../audit_logging'; import { getFilteredInstallPackages } from '../filtered_packages'; -import { _stateMachineInstallPackage } from './_state_machine_package_install'; +import { _stateMachineInstallPackage } from './install_state_machine/_state_machine_package_install'; import { formatVerificationResultForSO } from './package_verification'; import { getInstallation, getInstallationObject } from './get'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts similarity index 91% rename from x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts rename to x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts index e7a839981c9b7..10f87bfef692a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts @@ -14,36 +14,36 @@ import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/serv import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; -import { PackageSavedObjectConflictError } from '../../../errors'; +import { PackageSavedObjectConflictError } from '../../../../errors'; -import type { Installation } from '../../../../common'; +import type { Installation } from '../../../../../common'; -import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; -import { appContextService } from '../../app_context'; -import { createAppContextStartContractMock } from '../../../mocks'; -import { saveArchiveEntriesFromAssetsMap } from '../archive/storage'; -import { installILMPolicy } from '../elasticsearch/ilm/install'; -import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; +import { appContextService } from '../../../app_context'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { saveArchiveEntriesFromAssetsMap } from '../../archive/storage'; +import { installILMPolicy } from '../../elasticsearch/ilm/install'; +import { installIlmForDataStream } from '../../elasticsearch/datastream_ilm/install'; -jest.mock('../elasticsearch/template/template'); -jest.mock('../kibana/assets/install'); -jest.mock('../kibana/index_pattern/install'); -jest.mock('./install'); -jest.mock('./get'); -jest.mock('./install_index_template_pipeline'); +jest.mock('../../elasticsearch/template/template'); +jest.mock('../../kibana/assets/install'); +jest.mock('../../kibana/index_pattern/install'); +jest.mock('../install'); +jest.mock('../get'); +jest.mock('../install_index_template_pipeline'); -jest.mock('../archive/storage'); -jest.mock('../elasticsearch/ilm/install'); -jest.mock('../elasticsearch/datastream_ilm/install'); +jest.mock('../../archive/storage'); +jest.mock('../../elasticsearch/ilm/install'); +jest.mock('../../elasticsearch/datastream_ilm/install'); -import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; -import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; +import { updateCurrentWriteIndices } from '../../elasticsearch/template/template'; +import { installKibanaAssetsAndReferences } from '../../kibana/assets/install'; -import { MAX_TIME_COMPLETE_INSTALL } from '../../../../common/constants'; +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../../common/constants'; -import { restartInstallation } from './install'; -import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; +import { restartInstallation } from '../install'; +import { installIndexTemplatesAndPipelines } from '../install_index_template_pipeline'; import { _stateMachineInstallPackage } from './_state_machine_package_install'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts similarity index 92% rename from x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts rename to x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts index b882775c7e500..72366f757c05c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts @@ -15,11 +15,11 @@ import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server'; -import { PackageSavedObjectConflictError } from '../../../errors'; +import { PackageSavedObjectConflictError } from '../../../../errors'; -import type { HTTPAuthorizationHeader } from '../../../../common/http_authorization_header'; -import type { PackageInstallContext, StateNames, StateContext } from '../../../../common/types'; -import type { PackageAssetReference } from '../../../types'; +import type { HTTPAuthorizationHeader } from '../../../../../common/http_authorization_header'; +import type { PackageInstallContext, StateNames, StateContext } from '../../../../../common/types'; +import type { PackageAssetReference } from '../../../../types'; import type { Installation, @@ -30,7 +30,7 @@ import type { KibanaAssetReference, IndexTemplateEntry, AssetReference, -} from '../../../types'; +} from '../../../../types'; import { stepCreateRestartInstallation, @@ -45,9 +45,9 @@ import { stepSaveArchiveEntries, stepSaveSystemObject, updateLatestExecutedState, -} from './install_steps'; -import type { StateMachineDefinition } from './integrations_state_machine'; -import { handleState } from './integrations_state_machine'; +} from './steps'; +import type { StateMachineDefinition } from './state_machine'; +import { handleState } from './state_machine'; export interface InstallContext extends StateContext { savedObjectsClient: SavedObjectsClientContract; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts similarity index 99% rename from x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts rename to x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts index 179ff7e01a97c..df4e84e07ea3e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { createAppContextStartContractMock } from '../../../mocks'; -import { appContextService } from '../..'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../..'; -import { handleState } from './integrations_state_machine'; +import { handleState } from './state_machine'; const getTestDefinition = ( mockOnTransition1: any, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts similarity index 97% rename from x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts rename to x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts index 7cc56a8d3f15e..1aab829645a8e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/integrations_state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts @@ -4,8 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { appContextService } from '../../app_context'; -import type { StateContext, LatestExecutedState } from '../../../../common/types'; +import { appContextService } from '../../../app_context'; +import type { StateContext, LatestExecutedState } from '../../../../../common/types'; export interface State { onTransition: any; nextState?: string; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts new file mode 100644 index 0000000000000..8dc04ef8e8734 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './step_create_restart_installation'; +export * from './step_install_kibana_assets'; +export * from './step_install_ML_model'; +export * from './step_install_ilm_policies'; +export * from './step_install_index_template_pipelines'; +export * from './step_remove_legacy_templates'; +export * from './step_update_current_write_indices'; +export * from './step_install_transforms'; +export * from './step_delete_previous_pipelines'; +export * from './step_save_archive_entries'; +export * from './step_save_system_object'; +export * from './update_latest_executed_state'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts new file mode 100644 index 0000000000000..f7bfac1e97920 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ConcurrentInstallOperationError } from '../../../../../errors'; +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../../constants'; + +import { restartInstallation, createInstallation } from '../../install'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepCreateRestartInstallation(context: InstallContext) { + const { + savedObjectsClient, + logger, + installSource, + packageInstallContext, + spaceId, + force, + verificationResult, + installedPkg, + } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName, version: pkgVersion } = packageInfo; + // if some installation already exists + if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); + + // if the installation is currently running, don't try to install + // instead, only return already installed assets + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + logger.debug(`Package install - Installation is running and has exceeded timeout`); + + if (force) { + logger.debug(`Package install - Forced installation, restarting`); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } + } else { + // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL + // (it might be stuck) update the saved object and proceed + logger.debug( + `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` + ); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } + } else { + logger.debug(`Package install - Create installation`); + // step create_installation + await createInstallation({ + savedObjectsClient, + packageInfo, + installSource, + spaceId, + verificationResult, + }); + } + // Use a shared array that is updated by each operation. This allows each operation to accurately update the + // installation object with it's references without requiring a refresh of the SO index on each update (faster). + const esReferences = installedPkg?.attributes.installed_es ?? []; + return { esReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts new file mode 100644 index 0000000000000..e4fd50b52eb04 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + isTopLevelPipeline, + deletePreviousPipelines, +} from '../../../elasticsearch/ingest_pipeline'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepDeletePreviousPipelines(context: InstallContext) { + const { + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + installType, + installedPkg, + } = context; + const { packageInfo, paths } = packageInstallContext; + const { name: pkgName } = packageInfo; + let updatedESReferences; + // If this is an update or retrying an update, delete the previous version's pipelines + // Top-level pipeline assets will not be removed on upgrade as of ml model package addition which requires previous + // assets to remain installed. This is a temporary solution - more robust solution tracked here https://github.com/elastic/kibana/issues/115035 + if ( + paths.filter((path) => isTopLevelPipeline(path)).length === 0 && + (installType === 'update' || installType === 'reupdate') && + installedPkg + ) { + logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); + updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => + deletePreviousPipelines( + esClient, + savedObjectsClient, + pkgName, + installedPkg!.attributes.version, + esReferences || [] + ) + ); + } + // pipelines from a different version may have installed during a failed update + if (installType === 'rollback' && installedPkg) { + logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); + updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => + deletePreviousPipelines( + esClient, + savedObjectsClient, + pkgName, + installedPkg!.attributes.install_version, + esReferences || [] + ) + ); + } + return { esReferences: updatedESReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ML_model.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ML_model.ts new file mode 100644 index 0000000000000..5a43dd0a936a5 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ML_model.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installMlModel } from '../../../elasticsearch/ml_model'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallMlModel(context: InstallContext) { + const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; + + const updatedEsReferences = await withPackageSpan('Install ML models', () => + installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences || []) + ); + return { esReferences: updatedEsReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts new file mode 100644 index 0000000000000..95e80f6f461b6 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EsAssetReference } from '../../../../../types'; + +import { appContextService } from '../../../..'; + +import { installIlmForDataStream } from '../../../elasticsearch/datastream_ilm/install'; +import { installILMPolicy } from '../../../elasticsearch/ilm/install'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallILMPolicies(context: InstallContext) { + const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; + let updatedEsReferences: EsAssetReference[] = []; + + // currently only the base package has an ILM policy + // at some point ILM policies can be installed/modified + // per data stream and we should then save them + const isILMPoliciesDisabled = + appContextService.getConfig()?.internal?.disableILMPolicies ?? false; + if (!isILMPoliciesDisabled) { + updatedEsReferences = await withPackageSpan('Install ILM policies', () => + installILMPolicy( + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences || [] + ) + ); + + const res = await withPackageSpan('Install Data Stream ILM policies', () => + installIlmForDataStream( + packageInstallContext, + esClient, + savedObjectsClient, + logger, + updatedEsReferences + ) + ); + return { esReferences: res.esReferences }; + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts new file mode 100644 index 0000000000000..2d27063c518e1 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNormalizedDataStreams } from '../../../../../../common/services'; + +import { installIndexTemplatesAndPipelines } from '../../install_index_template_pipeline'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallIndexTemplatePipelines(context: InstallContext) { + const { + esClient, + savedObjectsClient, + packageInstallContext, + logger, + installedPkg, + esReferences, + } = context; + const { packageInfo } = packageInstallContext; + + if (packageInfo.type === 'integration') { + const { installedTemplates, esReferences: templateEsReferences } = + await installIndexTemplatesAndPipelines({ + installedPkg: installedPkg ? installedPkg.attributes : undefined, + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences: esReferences || [], + }); + return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; + } + + if (packageInfo.type === 'input' && installedPkg) { + // input packages create their data streams during package policy creation + // we must use installed_es to infer which streams exist first then + // we can install the new index templates + logger.debug(`Package install - packageInfo.type ${packageInfo.type}`); + const dataStreamNames = installedPkg.attributes.installed_es + .filter((ref) => ref.type === 'index_template') + // index templates are named {type}-{dataset}, remove everything before first hyphen + .map((ref) => ref.id.replace(/^[^-]+-/, '')); + + const dataStreams = dataStreamNames.flatMap((dataStreamName) => + getNormalizedDataStreams(packageInfo, dataStreamName) + ); + + if (dataStreams.length) { + const { installedTemplates, esReferences: templateEsReferences } = + await installIndexTemplatesAndPipelines({ + installedPkg: installedPkg ? installedPkg.attributes : undefined, + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences: esReferences || [], + onlyForDataStreams: dataStreams, + }); + return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; + } + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts new file mode 100644 index 0000000000000..56649c04428ac --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installKibanaAssetsAndReferences } from '../../../kibana/assets/install'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallKibanaAssets(context: InstallContext) { + const { + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + logger, + installedPkg, + packageInstallContext, + spaceId, + } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName, title: pkgTitle } = packageInfo; + + const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => + installKibanaAssetsAndReferences({ + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + pkgName, + pkgTitle, + packageInstallContext, + installedPkg, + logger, + spaceId, + assetTags: packageInfo?.asset_tags, + }) + ); + // Necessary to avoid async promise rejection warning + // See https://stackoverflow.com/questions/40920179/should-i-refrain-from-handling-promise-rejection-asynchronously + kibanaAssetPromise.catch(() => {}); + + return { kibanaAssetPromise }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts new file mode 100644 index 0000000000000..31474cfbf5bd8 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installTransforms } from '../../../elasticsearch/transform/install'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallTransforms(context: InstallContext) { + const { + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + force, + authorizationHeader, + } = context; + + const res = await withPackageSpan('Install transforms', () => + installTransforms({ + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + force, + authorizationHeader, + }) + ); + + return { esReferences: res.esReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts new file mode 100644 index 0000000000000..0c70989a67096 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { removeLegacyTemplates } from '../../../elasticsearch/template/remove_legacy'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepRemoveLegacyTemplates(context: InstallContext) { + const { esClient, packageInstallContext, logger } = context; + const { packageInfo } = packageInstallContext; + try { + await removeLegacyTemplates({ packageInfo, esClient, logger }); + } catch (e) { + logger.warn(`Error removing legacy templates: ${e.message}`); + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts new file mode 100644 index 0000000000000..d10a2a01b4565 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSETS_SAVED_OBJECT_TYPE } from '../../../../../constants'; +import type { PackageAssetReference } from '../../../../../types'; + +import { saveArchiveEntriesFromAssetsMap } from '../../../archive/storage'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepSaveArchiveEntries(context: InstallContext) { + const { packageInstallContext, savedObjectsClient, installSource, kibanaAssetPromise } = context; + const installedKibanaAssetsRefs = await kibanaAssetPromise; + + const { packageInfo } = packageInstallContext; + + const packageAssetResults = await withPackageSpan('Update archive entries', () => + saveArchiveEntriesFromAssetsMap({ + savedObjectsClient, + assetsMap: packageInstallContext.assetsMap, + paths: packageInstallContext.paths, + packageInfo, + installSource, + }) + ); + const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map( + (result) => ({ + id: result.id, + type: ASSETS_SAVED_OBJECT_TYPE, + }) + ); + + return { packageAssetRefs, installedKibanaAssetsRefs }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts new file mode 100644 index 0000000000000..f7bca891da6f7 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + PACKAGES_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, + FLEET_INSTALL_FORMAT_VERSION, +} from '../../../../../constants'; +import type { Installation } from '../../../../../types'; + +import { packagePolicyService } from '../../../..'; + +import { auditLoggingService } from '../../../../audit_logging'; + +import { withPackageSpan } from '../../utils'; + +import { clearLatestFailedAttempts } from '../../install_errors_helpers'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepSaveSystemObject(context: InstallContext) { + const { + packageInstallContext, + savedObjectsClient, + logger, + esClient, + installedPkg, + packageAssetRefs, + } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName, version: pkgVersion } = packageInfo; + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + + await withPackageSpan('Update install status', () => + savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + version: pkgVersion, + install_version: pkgVersion, + install_status: 'installed', + package_assets: packageAssetRefs, + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, + latest_install_failed_attempts: clearLatestFailedAttempts( + pkgVersion, + installedPkg?.attributes.latest_install_failed_attempts ?? [] + ), + }) + ); + + // Need to refetch the installation again to retrieve all the attributes + const updatedPackage = await savedObjectsClient.get( + PACKAGES_SAVED_OBJECT_TYPE, + pkgName + ); + logger.debug(`Package install - Install status ${updatedPackage?.attributes?.install_status}`); + // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its + // associated package policies after installation + if (updatedPackage.attributes.keep_policies_up_to_date) { + await withPackageSpan('Upgrade package policies', async () => { + const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, { + page: 1, + perPage: SO_SEARCH_LIMIT, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`, + }); + logger.debug( + `Package install - Package is flagged with keep_policies_up_to_date, upgrading its associated package policies ${policyIdsToUpgrade}` + ); + await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); + }); + } + logger.debug( + `Install status ${updatedPackage?.attributes?.install_status} - Installation complete!` + ); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts new file mode 100644 index 0000000000000..094f1110d9021 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { updateCurrentWriteIndices } from '../../../elasticsearch/template/template'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepUpdateCurrentWriteIndices(context: InstallContext) { + const { esClient, logger, ignoreMappingUpdateErrors, skipDataStreamRollover, indexTemplates } = + context; + + // update current backing indices of each data stream + await withPackageSpan('Update write indices', () => + updateCurrentWriteIndices(esClient, logger, indexTemplates || [], { + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }) + ); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts new file mode 100644 index 0000000000000..619c18a3f8373 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../constants'; + +import { auditLoggingService } from '../../../../audit_logging'; + +import type { InstallContext } from '../_state_machine_package_install'; + +// Function invoked after each transition +export const updateLatestExecutedState = async (context: InstallContext) => { + const { logger, savedObjectsClient, packageInstallContext, latestExecutedState } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName } = packageInfo; + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + try { + return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + latest_executed_state: latestExecutedState, + }); + } catch (err) { + if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { + logger.error(`failed to update package install state to: latest_executed_state ${err}`); + } + } +}; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts deleted file mode 100644 index b6e996ad5b1d3..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SavedObjectsErrorHelpers } from '@kbn/core/server'; - -import { ConcurrentInstallOperationError } from '../../../errors'; -import { - MAX_TIME_COMPLETE_INSTALL, - ASSETS_SAVED_OBJECT_TYPE, - PACKAGES_SAVED_OBJECT_TYPE, - PACKAGE_POLICY_SAVED_OBJECT_TYPE, - SO_SEARCH_LIMIT, - FLEET_INSTALL_FORMAT_VERSION, -} from '../../../constants'; -import type { PackageAssetReference, Installation, EsAssetReference } from '../../../types'; - -import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; - -import { appContextService, packagePolicyService } from '../..'; - -import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; -import { installILMPolicy } from '../elasticsearch/ilm/install'; -import { installMlModel } from '../elasticsearch/ml_model'; - -import { getNormalizedDataStreams } from '../../../../common/services'; - -import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy'; - -import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; - -import { installTransforms } from '../elasticsearch/transform/install'; - -import { isTopLevelPipeline, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline'; - -import { auditLoggingService } from '../../audit_logging'; - -import { saveArchiveEntriesFromAssetsMap } from '../archive/storage'; - -import { restartInstallation, createInstallation } from './install'; -import { withPackageSpan } from './utils'; -import type { InstallContext } from './_state_machine_package_install'; -import { installIndexTemplatesAndPipelines } from './install_index_template_pipeline'; -import { clearLatestFailedAttempts } from './install_errors_helpers'; - -export async function stepCreateRestartInstallation(context: InstallContext) { - const { - savedObjectsClient, - logger, - installSource, - packageInstallContext, - spaceId, - force, - verificationResult, - installedPkg, - } = context; - const { packageInfo } = packageInstallContext; - const { name: pkgName, version: pkgVersion } = packageInfo; - // if some installation already exists - if (installedPkg) { - const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; - const hasExceededTimeout = - Date.now() - Date.parse(installedPkg.attributes.install_started_at) < - MAX_TIME_COMPLETE_INSTALL; - logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); - - // if the installation is currently running, don't try to install - // instead, only return already installed assets - if (isStatusInstalling && hasExceededTimeout) { - // If this is a forced installation, ignore the timeout and restart the installation anyway - logger.debug(`Package install - Installation is running and has exceeded timeout`); - - if (force) { - logger.debug(`Package install - Forced installation, restarting`); - await restartInstallation({ - savedObjectsClient, - pkgName, - pkgVersion, - installSource, - verificationResult, - }); - } else { - throw new ConcurrentInstallOperationError( - `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ - pkgVersion || 'unknown' - } detected, aborting.` - ); - } - } else { - // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL - // (it might be stuck) update the saved object and proceed - logger.debug( - `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` - ); - await restartInstallation({ - savedObjectsClient, - pkgName, - pkgVersion, - installSource, - verificationResult, - }); - } - } else { - logger.debug(`Package install - Create installation`); - // step create_installation - await createInstallation({ - savedObjectsClient, - packageInfo, - installSource, - spaceId, - verificationResult, - }); - } - // Use a shared array that is updated by each operation. This allows each operation to accurately update the - // installation object with it's references without requiring a refresh of the SO index on each update (faster). - const esReferences = installedPkg?.attributes.installed_es ?? []; - return { esReferences }; -} - -export async function stepInstallKibanaAssets(context: InstallContext) { - const { - savedObjectsClient, - savedObjectsImporter, - savedObjectTagAssignmentService, - savedObjectTagClient, - logger, - installedPkg, - packageInstallContext, - spaceId, - } = context; - const { packageInfo } = packageInstallContext; - const { name: pkgName, title: pkgTitle } = packageInfo; - - const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => - installKibanaAssetsAndReferences({ - savedObjectsClient, - savedObjectsImporter, - savedObjectTagAssignmentService, - savedObjectTagClient, - pkgName, - pkgTitle, - packageInstallContext, - installedPkg, - logger, - spaceId, - assetTags: packageInfo?.asset_tags, - }) - ); - // Necessary to avoid async promise rejection warning - // See https://stackoverflow.com/questions/40920179/should-i-refrain-from-handling-promise-rejection-asynchronously - kibanaAssetPromise.catch(() => {}); - - return { kibanaAssetPromise }; -} - -export async function stepInstallILMPolicies(context: InstallContext) { - const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; - let updatedEsReferences: EsAssetReference[] = []; - - // currently only the base package has an ILM policy - // at some point ILM policies can be installed/modified - // per data stream and we should then save them - const isILMPoliciesDisabled = - appContextService.getConfig()?.internal?.disableILMPolicies ?? false; - if (!isILMPoliciesDisabled) { - updatedEsReferences = await withPackageSpan('Install ILM policies', () => - installILMPolicy( - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences || [] - ) - ); - - const res = await withPackageSpan('Install Data Stream ILM policies', () => - installIlmForDataStream( - packageInstallContext, - esClient, - savedObjectsClient, - logger, - updatedEsReferences - ) - ); - return { esReferences: res.esReferences }; - } -} - -export async function stepInstallMlModel(context: InstallContext) { - const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; - - const updatedEsReferences = await withPackageSpan('Install ML models', () => - installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences || []) - ); - return { esReferences: updatedEsReferences }; -} - -export async function stepInstallIndexTemplatePipelines(context: InstallContext) { - const { - esClient, - savedObjectsClient, - packageInstallContext, - logger, - installedPkg, - esReferences, - } = context; - const { packageInfo } = packageInstallContext; - - if (packageInfo.type === 'integration') { - const { installedTemplates, esReferences: templateEsReferences } = - await installIndexTemplatesAndPipelines({ - installedPkg: installedPkg ? installedPkg.attributes : undefined, - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences: esReferences || [], - }); - return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; - } - - if (packageInfo.type === 'input' && installedPkg) { - // input packages create their data streams during package policy creation - // we must use installed_es to infer which streams exist first then - // we can install the new index templates - logger.debug(`Package install - packageInfo.type ${packageInfo.type}`); - const dataStreamNames = installedPkg.attributes.installed_es - .filter((ref) => ref.type === 'index_template') - // index templates are named {type}-{dataset}, remove everything before first hyphen - .map((ref) => ref.id.replace(/^[^-]+-/, '')); - - const dataStreams = dataStreamNames.flatMap((dataStreamName) => - getNormalizedDataStreams(packageInfo, dataStreamName) - ); - - if (dataStreams.length) { - const { installedTemplates, esReferences: templateEsReferences } = - await installIndexTemplatesAndPipelines({ - installedPkg: installedPkg ? installedPkg.attributes : undefined, - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences: esReferences || [], - onlyForDataStreams: dataStreams, - }); - return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; - } - } -} - -export async function stepRemoveLegacyTemplates(context: InstallContext) { - const { esClient, packageInstallContext, logger } = context; - const { packageInfo } = packageInstallContext; - try { - await removeLegacyTemplates({ packageInfo, esClient, logger }); - } catch (e) { - logger.warn(`Error removing legacy templates: ${e.message}`); - } -} - -export async function stepUpdateCurrentWriteIndices(context: InstallContext) { - const { esClient, logger, ignoreMappingUpdateErrors, skipDataStreamRollover, indexTemplates } = - context; - - // update current backing indices of each data stream - await withPackageSpan('Update write indices', () => - updateCurrentWriteIndices(esClient, logger, indexTemplates || [], { - ignoreMappingUpdateErrors, - skipDataStreamRollover, - }) - ); -} - -export async function stepInstallTransforms(context: InstallContext) { - const { - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences, - force, - authorizationHeader, - } = context; - - const res = await withPackageSpan('Install transforms', () => - installTransforms({ - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences, - force, - authorizationHeader, - }) - ); - - return { esReferences: res.esReferences }; -} - -export async function stepDeletePreviousPipelines(context: InstallContext) { - const { - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences, - installType, - installedPkg, - } = context; - const { packageInfo, paths } = packageInstallContext; - const { name: pkgName } = packageInfo; - let updatedESReferences; - // If this is an update or retrying an update, delete the previous version's pipelines - // Top-level pipeline assets will not be removed on upgrade as of ml model package addition which requires previous - // assets to remain installed. This is a temporary solution - more robust solution tracked here https://github.com/elastic/kibana/issues/115035 - if ( - paths.filter((path) => isTopLevelPipeline(path)).length === 0 && - (installType === 'update' || installType === 'reupdate') && - installedPkg - ) { - logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); - updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => - deletePreviousPipelines( - esClient, - savedObjectsClient, - pkgName, - installedPkg!.attributes.version, - esReferences || [] - ) - ); - } - // pipelines from a different version may have installed during a failed update - if (installType === 'rollback' && installedPkg) { - logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); - updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => - deletePreviousPipelines( - esClient, - savedObjectsClient, - pkgName, - installedPkg!.attributes.install_version, - esReferences || [] - ) - ); - } - return { esReferences: updatedESReferences }; -} - -export async function stepSaveArchiveEntries(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, installSource, kibanaAssetPromise } = context; - const installedKibanaAssetsRefs = await kibanaAssetPromise; - - const { packageInfo } = packageInstallContext; - - const packageAssetResults = await withPackageSpan('Update archive entries', () => - saveArchiveEntriesFromAssetsMap({ - savedObjectsClient, - assetsMap: packageInstallContext.assetsMap, - paths: packageInstallContext.paths, - packageInfo, - installSource, - }) - ); - const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map( - (result) => ({ - id: result.id, - type: ASSETS_SAVED_OBJECT_TYPE, - }) - ); - - return { packageAssetRefs, installedKibanaAssetsRefs }; -} - -export async function stepSaveSystemObject(context: InstallContext) { - const { - packageInstallContext, - savedObjectsClient, - logger, - esClient, - installedPkg, - packageAssetRefs, - } = context; - const { packageInfo } = packageInstallContext; - const { name: pkgName, version: pkgVersion } = packageInfo; - - auditLoggingService.writeCustomSoAuditLog({ - action: 'update', - id: pkgName, - savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, - }); - - await withPackageSpan('Update install status', () => - savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - version: pkgVersion, - install_version: pkgVersion, - install_status: 'installed', - package_assets: packageAssetRefs, - install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, - latest_install_failed_attempts: clearLatestFailedAttempts( - pkgVersion, - installedPkg?.attributes.latest_install_failed_attempts ?? [] - ), - }) - ); - - // Need to refetch the installation again to retrieve all the attributes - const updatedPackage = await savedObjectsClient.get( - PACKAGES_SAVED_OBJECT_TYPE, - pkgName - ); - logger.debug(`Package install - Install status ${updatedPackage?.attributes?.install_status}`); - // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its - // associated package policies after installation - if (updatedPackage.attributes.keep_policies_up_to_date) { - await withPackageSpan('Upgrade package policies', async () => { - const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, { - page: 1, - perPage: SO_SEARCH_LIMIT, - kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`, - }); - logger.debug( - `Package install - Package is flagged with keep_policies_up_to_date, upgrading its associated package policies ${policyIdsToUpgrade}` - ); - await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); - }); - } - logger.debug( - `Install status ${updatedPackage?.attributes?.install_status} - Installation complete!` - ); -} - -// Function invoked after each transition -export const updateLatestExecutedState = async (context: InstallContext) => { - const { logger, savedObjectsClient, packageInstallContext, latestExecutedState } = context; - const { packageInfo } = packageInstallContext; - const { name: pkgName } = packageInfo; - - auditLoggingService.writeCustomSoAuditLog({ - action: 'update', - id: pkgName, - savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, - }); - try { - return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - latest_executed_state: latestExecutedState, - }); - } catch (err) { - if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { - logger.error(`failed to update package install state to: latest_executed_state ${err}`); - } - } -}; From e9ee148707766620891610fdf5f1384cacf7821b Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 29 Mar 2024 12:40:27 +0100 Subject: [PATCH 25/38] Enable feature flag for integration tests --- x-pack/test/fleet_api_integration/config.base.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/fleet_api_integration/config.base.ts b/x-pack/test/fleet_api_integration/config.base.ts index 5626ee4d85d6e..fd9d8e08779c0 100644 --- a/x-pack/test/fleet_api_integration/config.base.ts +++ b/x-pack/test/fleet_api_integration/config.base.ts @@ -76,6 +76,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { 'agentTamperProtectionEnabled', 'enableStrictKQLValidation', 'subfeaturePrivileges', + 'enablePackagesStateMachine', ])}`, `--logging.loggers=${JSON.stringify([ ...getKibanaCliLoggers(xPackAPITestsConfig.get('kbnTestServer.serverArgs')), From 28edd3b7d124f8a6bed58574065912643aacf99f Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 29 Mar 2024 12:57:18 +0100 Subject: [PATCH 26/38] Fix filename --- .../services/epm/packages/install_state_machine/steps/index.ts | 2 +- .../steps/{step_install_ML_model.ts => step_install_mlmodel.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/{step_install_ML_model.ts => step_install_mlmodel.ts} (100%) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts index 8dc04ef8e8734..3fb2074a4b20a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts @@ -7,7 +7,7 @@ export * from './step_create_restart_installation'; export * from './step_install_kibana_assets'; -export * from './step_install_ML_model'; +export * from './step_install_mlmodel'; export * from './step_install_ilm_policies'; export * from './step_install_index_template_pipelines'; export * from './step_remove_legacy_templates'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ML_model.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts similarity index 100% rename from x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ML_model.ts rename to x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts From 4a16aeb68782f9dbf9d8fa9da18d183e3c14dae5 Mon Sep 17 00:00:00 2001 From: criamico Date: Tue, 2 Apr 2024 10:55:59 +0200 Subject: [PATCH 27/38] Fix missing assets return --- .../_state_machine_package_install.ts | 15 ++++----------- .../steps/step_delete_previous_pipelines.ts | 8 +++++--- .../steps/step_install_ilm_policies.ts | 2 +- .../step_install_index_template_pipelines.ts | 10 ++++++++-- .../steps/step_install_mlmodel.ts | 2 +- .../steps/step_install_transforms.ts | 3 +-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts index 72366f757c05c..3b1760d2e343e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts @@ -143,17 +143,10 @@ export async function _stateMachineInstallPackage( installStates, installStates.context ); - if ( - installedKibanaAssetsRefs && - installedKibanaAssetsRefs.length && - esReferences && - esReferences.length - ) - return [ - ...(installedKibanaAssetsRefs as KibanaAssetReference[]), - ...(esReferences as EsAssetReference[]), - ]; - return []; + return [ + ...(installedKibanaAssetsRefs as KibanaAssetReference[]), + ...(esReferences as EsAssetReference[]), + ]; } catch (err) { const { packageInfo } = installStates.context.packageInstallContext; const { name: pkgName, version: pkgVersion } = packageInfo; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts index e4fd50b52eb04..dc8957c871051 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts @@ -45,9 +45,8 @@ export async function stepDeletePreviousPipelines(context: InstallContext) { esReferences || [] ) ); - } - // pipelines from a different version may have installed during a failed update - if (installType === 'rollback' && installedPkg) { + } else if (installType === 'rollback' && installedPkg) { + // pipelines from a different version may have been installed during a failed update logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => deletePreviousPipelines( @@ -58,6 +57,9 @@ export async function stepDeletePreviousPipelines(context: InstallContext) { esReferences || [] ) ); + } else { + // if none of the previous cases, return the original esRefences + updatedESReferences = esReferences; } return { esReferences: updatedESReferences }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts index 95e80f6f461b6..6c19517085757 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts @@ -45,6 +45,6 @@ export async function stepInstallILMPolicies(context: InstallContext) { updatedEsReferences ) ); - return { esReferences: res.esReferences }; + return { esReferences: res?.esReferences || esReferences }; } } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts index 2d27063c518e1..4eff81c691ed9 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts @@ -23,6 +23,9 @@ export async function stepInstallIndexTemplatePipelines(context: InstallContext) const { packageInfo } = packageInstallContext; if (packageInfo.type === 'integration') { + logger.debug( + `Package install - Installing index templates and pipelines, packageInfo.type: ${packageInfo.type}` + ); const { installedTemplates, esReferences: templateEsReferences } = await installIndexTemplatesAndPipelines({ installedPkg: installedPkg ? installedPkg.attributes : undefined, @@ -32,14 +35,17 @@ export async function stepInstallIndexTemplatePipelines(context: InstallContext) logger, esReferences: esReferences || [], }); - return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; + return { + esReferences: templateEsReferences || esReferences, + indexTemplates: installedTemplates, + }; } if (packageInfo.type === 'input' && installedPkg) { // input packages create their data streams during package policy creation // we must use installed_es to infer which streams exist first then // we can install the new index templates - logger.debug(`Package install - packageInfo.type ${packageInfo.type}`); + logger.debug(`Package install - packageInfo.type: ${packageInfo.type}`); const dataStreamNames = installedPkg.attributes.installed_es .filter((ref) => ref.type === 'index_template') // index templates are named {type}-{dataset}, remove everything before first hyphen diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts index 5a43dd0a936a5..a121d543f8d7a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts @@ -17,5 +17,5 @@ export async function stepInstallMlModel(context: InstallContext) { const updatedEsReferences = await withPackageSpan('Install ML models', () => installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences || []) ); - return { esReferences: updatedEsReferences }; + return { esReferences: updatedEsReferences || esReferences }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts index 31474cfbf5bd8..e04f831dd28d2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts @@ -33,6 +33,5 @@ export async function stepInstallTransforms(context: InstallContext) { authorizationHeader, }) ); - - return { esReferences: res.esReferences }; + return { esReferences: res.esReferences || esReferences }; } From 741065fcb9d13a30e036bc1b2b336000e4d8b81a Mon Sep 17 00:00:00 2001 From: criamico Date: Tue, 2 Apr 2024 11:41:43 +0200 Subject: [PATCH 28/38] Fix postTransition call --- .../state_machine.test.ts | 20 ++++++++++++++++--- .../install_state_machine/state_machine.ts | 9 ++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts index df4e84e07ea3e..3e7494712c5b3 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts @@ -421,7 +421,7 @@ describe('handleState', () => { ); }); - it('should not execute postTransition when a transition exits with errors', async () => { + it('should execute postTransition correctly also when a transition throws', async () => { const error = new Error('Installation failed'); const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); const mockOnTransitionState2 = jest.fn().mockRejectedValue(error); @@ -436,7 +436,19 @@ describe('handleState', () => { mockPostTransition ); try { - await handleState('state1', testDefinition, testDefinition.context); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + error: + 'Error during execution of state "state2" with status "failed": Installation failed', + }, + }) + ); } catch (err) { expect(err).toEqual( `Error during execution of state \"state2\" with status \"failed\": Installation failed` @@ -450,6 +462,8 @@ describe('handleState', () => { latestExecutedState: { name: 'state1', started_at: expect.anything(), + error: + 'Error during execution of state "state2" with status "failed": Installation failed', }, }) ); @@ -467,7 +481,7 @@ describe('handleState', () => { expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); }); - it('should log a warning when postTransition exits with erros and continue executing the states', async () => { + it('should log a warning when postTransition exits with errors and continue executing the states', async () => { const error = new Error('Installation failed'); const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); const mockOnTransitionState2 = jest.fn(); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts index 1aab829645a8e..d7534db18a10c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import { appContextService } from '../../../app_context'; import type { StateContext, LatestExecutedState } from '../../../../../common/types'; export interface State { @@ -69,6 +70,12 @@ export async function handleState( updatedContext = { ...updatedContext, latestExecutedState: latestStateWithError }; logger.warn(errorMessage); + // execute post transition function when transition failed too + if (typeof currentState.onPostTransition === 'function') { + await currentState.onPostTransition.call(undefined, updatedContext); + logger.debug(`Executing post transition function: ${currentState.onPostTransition.name}`); + } + // bubble up the error throw errorMessage; } @@ -78,7 +85,7 @@ export async function handleState( `Execution of state "${currentStateName}" with status "${currentStatus}": provided onTransition is not a valid function` ); } - + // execute post transition function if (typeof currentState.onPostTransition === 'function') { try { await currentState.onPostTransition.call(undefined, updatedContext); From 6f7809c35c97ce8d74be3169e7f27027ab88df4b Mon Sep 17 00:00:00 2001 From: criamico Date: Tue, 2 Apr 2024 17:06:45 +0200 Subject: [PATCH 29/38] Address code review comments --- .../plugins/fleet/common/types/models/epm.ts | 32 +++++++++---------- .../_state_machine_package_install.ts | 3 +- .../state_machine.test.ts | 18 +++++++++-- .../install_state_machine/state_machine.ts | 27 ++++++++++------ .../steps/update_latest_executed_state.ts | 15 +++++---- .../apis/epm/install_remove_assets.ts | 4 +++ 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 87435c03eb468..176b42d7cf990 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -549,22 +549,22 @@ export interface InstallFailedAttempt { }; } -const installStateNames = [ - 'create_restart_installation', - 'install_kibana_assets', - 'install_ilm_policies', - 'install_ml_model', - 'install_index_template_pipelines', - 'remove_legacy_templates', - 'update_current_write_indices', - 'install_transforms', - 'delete_previous_pipelines', - 'save_archive_entries_from_assets_map', - 'update_so', -] as const; - -type StateNamesTuple = typeof installStateNames; -export type StateNames = StateNamesTuple[number]; +export enum INSTALL_STATES { + CREATE_RESTART_INSTALLATION = 'create_restart_installation', + INSTALL_KIBANA_ASSETS = 'install_kibana_assets', + INSTALL_ILM_POLICIES = 'install_ilm_policies', + INSTALL_ML_MODEL = 'install_ml_model', + INSTALL_INDEX_TEMPLATE_PIPELINES = 'install_index_template_pipelines', + REMOVE_LEGACY_TEMPLATES = 'remove_legacy_templates', + UPDATE_CURRENT_WRITE_INDICES = 'update_current_write_indices', + INSTALL_TRANSFORMS = 'install_transforms', + DELETE_PREVIOUS_PIPELINES = 'delete_previous_pipelines', + SAVE_ARCHIVE_ENTRIES = 'save_archive_entries_from_assets_map', + UPDATE_SO = 'update_so', +} +type StatesKeys = keyof typeof INSTALL_STATES; +export type StateNames = typeof INSTALL_STATES[StatesKeys]; + export interface LatestExecutedState { name: T; started_at: string; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts index 3b1760d2e343e..79f82343cb332 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts @@ -18,6 +18,7 @@ import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging import { PackageSavedObjectConflictError } from '../../../../errors'; import type { HTTPAuthorizationHeader } from '../../../../../common/http_authorization_header'; +import { INSTALL_STATES } from '../../../../../common/types'; import type { PackageInstallContext, StateNames, StateContext } from '../../../../../common/types'; import type { PackageAssetReference } from '../../../../types'; @@ -139,7 +140,7 @@ export async function _stateMachineInstallPackage( }; try { const { installedKibanaAssetsRefs, esReferences } = await handleState( - 'create_restart_installation', + INSTALL_STATES.CREATE_RESTART_INSTALLATION, installStates, installStates.context ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts index 3e7494712c5b3..493021a76f16c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts @@ -333,18 +333,30 @@ describe('handleState', () => { ); }); - it('should throw when a state returns error', async () => { + it('should throw and return updated context with latest error when a state returns error', async () => { const error = new Error('Installation failed'); const mockOnTransitionState1 = jest.fn().mockRejectedValue(error); const mockOnTransitionState2 = jest.fn(); const mockOnTransitionState3 = jest.fn(); + const contextData = { fixedVal: 'something' }; const testDefinition = getTestDefinition( mockOnTransitionState1, mockOnTransitionState2, - mockOnTransitionState3 + mockOnTransitionState3, + contextData ); try { - await handleState('state1', testDefinition, testDefinition.context); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + expect(updatedContext).toEqual( + expect.objectContaining({ + fixedVal: 'something', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + error: `Error during execution of state "state1" with status "failed": Installation failed`, + }, + }) + ); } catch (err) { expect(err).toEqual( `Error during execution of state "state1" with status "failed": Installation failed` diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts index d7534db18a10c..bb3dc3e377c67 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts @@ -5,6 +5,8 @@ * 2.0. */ +import type { Logger } from '@kbn/core/server'; + import { appContextService } from '../../../app_context'; import type { StateContext, LatestExecutedState } from '../../../../../common/types'; export interface State { @@ -61,6 +63,7 @@ export async function handleState( `Executed state: ${currentStateName} with status: ${currentStatus} - nextState: ${currentState.nextState}` ); } catch (error) { + console.log('## error', error); currentStatus = 'failed'; const errorMessage = `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}`; const latestStateWithError = { @@ -71,10 +74,7 @@ export async function handleState( logger.warn(errorMessage); // execute post transition function when transition failed too - if (typeof currentState.onPostTransition === 'function') { - await currentState.onPostTransition.call(undefined, updatedContext); - logger.debug(`Executing post transition function: ${currentState.onPostTransition.name}`); - } + await executePostTransition(logger, updatedContext, currentState); // bubble up the error throw errorMessage; @@ -86,6 +86,20 @@ export async function handleState( ); } // execute post transition function + await executePostTransition(logger, updatedContext, currentState); + + if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { + return await handleState(currentState.nextState, definition, updatedContext); + } else { + return updatedContext; + } +} + +async function executePostTransition( + logger: Logger, + updatedContext: StateContext, + currentState: State +) { if (typeof currentState.onPostTransition === 'function') { try { await currentState.onPostTransition.call(undefined, updatedContext); @@ -94,9 +108,4 @@ export async function handleState( logger.warn(`Error during execution of post transition function: ${error.message}`); } } - if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { - return await handleState(currentState.nextState, definition, updatedContext); - } else { - return updatedContext; - } } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts index 619c18a3f8373..54b64431d775c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts @@ -19,18 +19,21 @@ export const updateLatestExecutedState = async (context: InstallContext) => { const { packageInfo } = packageInstallContext; const { name: pkgName } = packageInfo; - auditLoggingService.writeCustomSoAuditLog({ - action: 'update', - id: pkgName, - savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, - }); try { + // If the error is of type ConcurrentInstallationError, don't save it in the SO + if (latestExecutedState?.error?.includes('Concurrent installation or upgrade')) return; + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { latest_executed_state: latestExecutedState, }); } catch (err) { if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { - logger.error(`failed to update package install state to: latest_executed_state ${err}`); + logger.error(`Failed to update package install state to: latest_executed_state ${err}`); } } }; diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 9c0b5cd8a426e..96e5e95e720ad 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -778,6 +778,10 @@ const expectAssetsInstalled = ({ install_started_at: res.attributes.install_started_at, install_source: 'registry', latest_install_failed_attempts: [], + latest_executed_state: { + name: 'update_so', + started_at: res.attributes.latest_executed_state.started_at, + }, install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, verification_status: 'unknown', verification_key_id: null, From 1beb1bbd7e34c3fbcb49d6be8fa77160b2de386f Mon Sep 17 00:00:00 2001 From: criamico Date: Tue, 2 Apr 2024 17:08:54 +0200 Subject: [PATCH 30/38] remove console log --- .../services/epm/packages/install_state_machine/state_machine.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts index bb3dc3e377c67..501bb0baf591f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts @@ -63,7 +63,6 @@ export async function handleState( `Executed state: ${currentStateName} with status: ${currentStatus} - nextState: ${currentState.nextState}` ); } catch (error) { - console.log('## error', error); currentStatus = 'failed'; const errorMessage = `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}`; const latestStateWithError = { From dc95c4552b0571566ee88258c3ae82ae79102f89 Mon Sep 17 00:00:00 2001 From: criamico Date: Wed, 3 Apr 2024 12:55:36 +0200 Subject: [PATCH 31/38] fix integration test and add unit test --- .../update_latest_executed_state.test.ts | 195 ++++++++++++++++++ .../steps/update_latest_executed_state.ts | 2 +- .../apis/epm/update_assets.ts | 4 + 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts new file mode 100644 index 0000000000000..afce673348d7e --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObjectsUpdateResponse, +} from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { + MAX_TIME_COMPLETE_INSTALL, + PACKAGES_SAVED_OBJECT_TYPE, +} from '../../../../../../common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; + +import { INSTALL_STATES } from '../../../../../../common/types'; + +import { auditLoggingService } from '../../../../audit_logging'; + +import type { PackagePolicySOAttributes } from '../../../../../types'; + +import { updateLatestExecutedState } from './update_latest_executed_state'; + +jest.mock('../../../../audit_logging'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +describe('updateLatestExecutedState', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(() => { + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + soClient.update.mockReset(); + }); + + it('Updates the SO after each transition', async () => { + await updateLatestExecutedState({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual( + expect.objectContaining([ + [ + 'epm-packages', + 'xyz', + { + latest_executed_state: { + name: 'save_archive_entries_from_assets_map', + started_at: expect.anything(), + }, + }, + ], + ]) + ); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'xyz', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + }); + + it('Should not update the SO if the context contains concurrent installation error', async () => { + await updateLatestExecutedState({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + error: `Concurrent installation or upgrade of xyz-4.5.6 detected.`, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual([]); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).not.toHaveBeenCalled(); + }); + + it('Should log error if the update failed', async () => { + soClient.update.mockImplementation( + async ( + _type: string, + _id: string + ): Promise> => { + throw SavedObjectsErrorHelpers.createConflictError('abc', '123'); + } + ); + + await updateLatestExecutedState({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'xyz', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + expect(logger.error).toHaveBeenCalledWith( + 'Failed to update SO with latest executed state: Error: Saved object [abc/123] conflict' + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts index 54b64431d775c..55d7997ad58f7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts @@ -33,7 +33,7 @@ export const updateLatestExecutedState = async (context: InstallContext) => { }); } catch (err) { if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { - logger.error(`Failed to update package install state to: latest_executed_state ${err}`); + logger.error(`Failed to update SO with latest executed state: ${err}`); } } }; diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index cd3898a58c6a7..fe584f9cd04f7 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -486,6 +486,10 @@ export default function (providerContext: FtrProviderContext) { install_source: 'registry', install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, latest_install_failed_attempts: [], + latest_executed_state: { + name: 'update_so', + started_at: res.attributes.latest_executed_state.started_at, + }, verification_status: 'unknown', verification_key_id: null, }); From f503c1d99e0bd437f63ff5bb0d455b505172f048 Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 4 Apr 2024 15:50:29 +0200 Subject: [PATCH 32/38] Fix tests and enable feature flag --- .../fleet/common/experimental_features.ts | 2 +- .../plugins/fleet/common/types/models/epm.ts | 1 + .../_state_machine_package_install.test.ts | 205 ++++++++---------- .../_state_machine_package_install.ts | 6 + .../install_state_machine/state_machine.ts | 35 ++- .../install_state_machine/steps/index.ts | 1 + .../steps/step_create_restart_installation.ts | 7 +- .../steps/step_install_ilm_policies.ts | 7 +- .../steps/step_resolve_kibana_promise.ts | 15 ++ .../steps/step_save_archive_entries.ts | 5 +- .../apis/epm/install_error_rollback.ts | 5 +- 11 files changed, 165 insertions(+), 124 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index 44421348e3d72..48e77dbe1988d 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -28,7 +28,7 @@ export const allowedExperimentalValues = Object.freeze>( agentless: false, enableStrictKQLValidation: false, subfeaturePrivileges: false, - enablePackagesStateMachine: false, + enablePackagesStateMachine: true, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 176b42d7cf990..a62833dfdcfb5 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -560,6 +560,7 @@ export enum INSTALL_STATES { INSTALL_TRANSFORMS = 'install_transforms', DELETE_PREVIOUS_PIPELINES = 'delete_previous_pipelines', SAVE_ARCHIVE_ENTRIES = 'save_archive_entries_from_assets_map', + RESOLVE_KIBANA_PROMISE = 'resolve_kibana_promise', UPDATE_SO = 'update_so', } type StatesKeys = keyof typeof INSTALL_STATES; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts index 10f87bfef692a..2d4db5ceb2f23 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts @@ -14,7 +14,10 @@ import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/serv import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; -import { PackageSavedObjectConflictError } from '../../../../errors'; +import { + PackageSavedObjectConflictError, + ConcurrentInstallOperationError, +} from '../../../../errors'; import type { Installation } from '../../../../../common'; @@ -89,7 +92,7 @@ describe('_stateMachineInstallPackage', () => { jest.mocked(restartInstallation).mockReset(); }); - it('handles errors from installKibanaAssets', async () => { + it('Handles errors from installKibanaAssets', async () => { // force errors from this function mockedInstallKibanaAssetsAndReferences.mockImplementation(async () => { throw new Error('mocked async error A: should be caught'); @@ -105,43 +108,39 @@ describe('_stateMachineInstallPackage', () => { }); // use this workaround to test the error; toThrow/toThrowError doesn't match correctly - let thrownError; - try { - await _stateMachineInstallPackage({ - savedObjectsClient: soClient, - // @ts-ignore - savedObjectsImporter: jest.fn(), - esClient, - logger: loggerMock.create(), - packageInstallContext: { - assetsMap: new Map(), - paths: [], - packageInfo: { - title: 'title', - name: 'xyz', - version: '4.5.6', - description: 'test', - type: 'integration', - categories: ['cloud', 'custom'], - format_version: 'string', - release: 'experimental', - conditions: { kibana: { version: 'x.y.z' } }, - owner: { github: 'elastic/fleet' }, - }, + const installationPromise = _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, }, - installType: 'install', - installSource: 'registry', - spaceId: DEFAULT_SPACE_ID, - }); - } catch (error) { - thrownError = error; - } - expect(thrownError).toEqual( - `Error during execution of state \"save_archive_entries_from_assets_map\" with status \"failed\": mocked async error A: should be caught` - ); + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + // if we have a .catch this will fail nicely (test pass) + // otherwise the test will fail with either of the mocked errors + await expect(installationPromise).rejects.toThrow('mocked'); + await expect(installationPromise).rejects.toThrow('should be caught'); }); - it('do not install ILM policies if disabled in config', async () => { + it('Do not install ILM policies if disabled in config', async () => { appContextService.start( createAppContextStartContractMock({ internal: { @@ -198,7 +197,7 @@ describe('_stateMachineInstallPackage', () => { expect(installIlmForDataStream).not.toBeCalled(); }); - it('install ILM policies if not disabled in config', async () => { + it('Installs ILM policies if not disabled in config', async () => { appContextService.start( createAppContextStartContractMock({ internal: { @@ -255,7 +254,7 @@ describe('_stateMachineInstallPackage', () => { expect(installIlmForDataStream).toBeCalled(); }); - describe('when package is stuck in `installing`', () => { + describe('When package is stuck in `installing`', () => { const mockInstalledPackageSo: SavedObject = { id: 'mocked-package', attributes: { @@ -292,7 +291,7 @@ describe('_stateMachineInstallPackage', () => { ); }); - describe('timeout reached', () => { + describe('When timeout is reached', () => { it('restarts installation', async () => { await _stateMachineInstallPackage({ savedObjectsClient: soClient, @@ -324,45 +323,38 @@ describe('_stateMachineInstallPackage', () => { }); }); - describe('Timeout not reached', () => { - describe('Force flag not provided', () => { - it('throws concurrent installation error if force flag is not provided', async () => { - // use this workaround to test the error; toThrow/toThrowError doesn't match correctly - let thrownError; - try { - await _stateMachineInstallPackage({ - savedObjectsClient: soClient, - // @ts-ignore - savedObjectsImporter: jest.fn(), - esClient, - logger: loggerMock.create(), - packageInstallContext: { - paths: [], - assetsMap: new Map(), - packageInfo: { - name: mockInstalledPackageSo.attributes.name, - version: mockInstalledPackageSo.attributes.version, - title: mockInstalledPackageSo.attributes.name, - } as any, - }, - installedPkg: { - ...mockInstalledPackageSo, - attributes: { - ...mockInstalledPackageSo.attributes, - install_started_at: new Date(Date.now() - 1000).toISOString(), - }, + describe('When timeout is not reached', () => { + describe('With no force flag', () => { + it('throws concurrent installation error', async () => { + const installPromise = _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + paths: [], + assetsMap: new Map(), + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), }, - }); - } catch (error) { - thrownError = error; - } - expect(thrownError).toEqual( - `Error during execution of state \"create_restart_installation\" with status \"failed\": Concurrent installation or upgrade of test-package-1.0.0 detected, aborting.` - ); + }, + }); + + await expect(installPromise).rejects.toThrowError(ConcurrentInstallOperationError); }); }); - describe('force flag provided', () => { + describe('With force flag provided', () => { it('restarts installation', async () => { await _stateMachineInstallPackage({ savedObjectsClient: soClient, @@ -395,7 +387,7 @@ describe('_stateMachineInstallPackage', () => { }); }); - it('surfaces saved object conflicts error', async () => { + it('Surfaces saved object conflicts error', async () => { appContextService.start( createAppContextStartContractMock({ internal: { @@ -415,40 +407,33 @@ describe('_stateMachineInstallPackage', () => { mockedInstallKibanaAssetsAndReferences.mockRejectedValueOnce( new PackageSavedObjectConflictError('test') ); - let thrownError; - // use this workaround to test the error; toThrow/toThrowError doesn't match correctly - try { - await _stateMachineInstallPackage({ - savedObjectsClient: soClient, - // @ts-ignore - savedObjectsImporter: jest.fn(), - esClient, - logger: loggerMock.create(), - packageInstallContext: { - packageInfo: { - title: 'title', - name: 'xyz', - version: '4.5.6', - description: 'test', - type: 'integration', - categories: ['cloud', 'custom'], - format_version: 'string', - release: 'experimental', - conditions: { kibana: { version: 'x.y.z' } }, - owner: { github: 'elastic/fleet' }, - } as any, - assetsMap: new Map(), - paths: [], - }, - installType: 'install', - installSource: 'registry', - spaceId: DEFAULT_SPACE_ID, - }); - } catch (error) { - thrownError = error; - } - expect(thrownError).toEqual( - `Error during execution of state "save_archive_entries_from_assets_map" with status "failed": test` - ); + + const installPromise = _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + await expect(installPromise).rejects.toThrowError(PackageSavedObjectConflictError); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts index 79f82343cb332..e86beccec9e4a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts @@ -44,6 +44,7 @@ import { stepInstallTransforms, stepDeletePreviousPipelines, stepSaveArchiveEntries, + stepResolveKibanaPromise, stepSaveSystemObject, updateLatestExecutedState, } from './steps'; @@ -128,6 +129,11 @@ export async function _stateMachineInstallPackage( }, save_archive_entries_from_assets_map: { onTransition: stepSaveArchiveEntries, + nextState: 'resolve_kibana_promise', + onPostTransition: updateLatestExecutedState, + }, + resolve_kibana_promise: { + onTransition: stepResolveKibanaPromise, nextState: 'update_so', onPostTransition: updateLatestExecutedState, }, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts index 501bb0baf591f..c70a99e272361 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts @@ -18,11 +18,40 @@ export interface State { export type StatusName = 'success' | 'failed' | 'pending'; export type StateMachineStates = Record; +/* + * Data structure defining the state machine + * { + * context: {}, + * states: { + * state1: { + * onTransition: onState1Transition, + * onPostTransition: onPostTransition, + * nextState: 'state2', + * }, + * state2: { + * onTransition: onState2Transition, + * onPostTransition: onPostTransition,, + * nextState: 'state3', + * }, + * state3: { + * onTransition: onState3Transition, + * onPostTransition: onPostTransition, + * nextState: 'end', + * } + * } + */ export interface StateMachineDefinition { - context?: any; + context: StateContext; states: StateMachineStates; } - +/* + * Generic state machine implemented to handle state transitions, based on a provided data structure + * currentStateName: iniital state + * definition: data structure defined as a StateMachineDefinition + * context: object keeping the state between transitions. All the transition functions accept it as input parameter and write to it + * + * It recursively traverses all the states until it finds the last state. + */ export async function handleState( currentStateName: string, definition: StateMachineDefinition, @@ -76,7 +105,7 @@ export async function handleState( await executePostTransition(logger, updatedContext, currentState); // bubble up the error - throw errorMessage; + throw error; } } else { currentStatus = 'failed'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts index 3fb2074a4b20a..c34c4f566715b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts @@ -16,4 +16,5 @@ export * from './step_install_transforms'; export * from './step_delete_previous_pipelines'; export * from './step_save_archive_entries'; export * from './step_save_system_object'; +export * from './step_resolve_kibana_promise'; export * from './update_latest_executed_state'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts index f7bfac1e97920..58daa6c379134 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts @@ -25,6 +25,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) { } = context; const { packageInfo } = packageInstallContext; const { name: pkgName, version: pkgVersion } = packageInfo; + // if some installation already exists if (installedPkg) { const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; @@ -71,7 +72,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) { } } else { logger.debug(`Package install - Create installation`); - // step create_installation + await createInstallation({ savedObjectsClient, packageInfo, @@ -80,8 +81,4 @@ export async function stepCreateRestartInstallation(context: InstallContext) { verificationResult, }); } - // Use a shared array that is updated by each operation. This allows each operation to accurately update the - // installation object with it's references without requiring a refresh of the SO index on each update (faster). - const esReferences = installedPkg?.attributes.installed_es ?? []; - return { esReferences }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts index 6c19517085757..05ae59c712a8f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts @@ -17,7 +17,12 @@ import { withPackageSpan } from '../../utils'; import type { InstallContext } from '../_state_machine_package_install'; export async function stepInstallILMPolicies(context: InstallContext) { - const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; + const { logger, packageInstallContext, esClient, savedObjectsClient, installedPkg } = context; + + // Array that gets updated by each operation. This allows each operation to accurately update the + // installation object with its references without requiring a refresh of the SO index on each update (faster). + const esReferences = installedPkg?.attributes.installed_es ?? []; + let updatedEsReferences: EsAssetReference[] = []; // currently only the base package has an ILM policy diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts new file mode 100644 index 0000000000000..72782438c20b6 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepResolveKibanaPromise(context: InstallContext) { + const { kibanaAssetPromise } = context; + const installedKibanaAssetsRefs = await kibanaAssetPromise; + + return { installedKibanaAssetsRefs }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index d10a2a01b4565..3e4702c459eed 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -15,8 +15,7 @@ import { withPackageSpan } from '../../utils'; import type { InstallContext } from '../_state_machine_package_install'; export async function stepSaveArchiveEntries(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, installSource, kibanaAssetPromise } = context; - const installedKibanaAssetsRefs = await kibanaAssetPromise; + const { packageInstallContext, savedObjectsClient, installSource } = context; const { packageInfo } = packageInstallContext; @@ -36,5 +35,5 @@ export async function stepSaveArchiveEntries(context: InstallContext) { }) ); - return { packageAssetRefs, installedKibanaAssetsRefs }; + return { packageAssetRefs }; } diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts index 5192b8a4e914b..5f4c5b784a280 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts @@ -27,7 +27,10 @@ export default function (providerContext: FtrProviderContext) { }; const uninstallPackage = async (pkg: string, version: string) => { - await supertest.delete(`/api/fleet/epm/packages/${pkg}/${version}`).set('kbn-xsrf', 'xxxx'); + await supertest + .delete(`/api/fleet/epm/packages/${pkg}/${version}`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }); }; const getPackageInfo = async (pkg: string, version: string) => { From 44f192466b0ba206a4ead6ce40f1678944f1eb6b Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 4 Apr 2024 16:58:08 +0200 Subject: [PATCH 33/38] Unit test for create_restart_installation --- .../_state_machine_package_install.ts | 8 +- .../step_create_restart_installation.test.ts | 198 ++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts index e86beccec9e4a..d66334b315a42 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts @@ -75,7 +75,13 @@ export interface InstallContext extends StateContext { esReferences?: EsAssetReference[]; kibanaAssetPromise?: Promise; } - +/* + * _stateMachineInstallPackage installs packages using the generic state machine in ./state_machine + * installStates is the data structure providing the state machine definition + * Usually the install process starts with `create_restart_installation` and continues based on nextState parameter in the definition + * The `onTransition` functions are the steps executed to go from one state to another, and all accept an `InstallContext` object as input parameter + * After each transition `updateLatestExecutedState` is executed, it updates the executed state in the SO + */ export async function _stateMachineInstallPackage( context: InstallContext ): Promise { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts new file mode 100644 index 0000000000000..9323841daba00 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { + MAX_TIME_COMPLETE_INSTALL, + PACKAGES_SAVED_OBJECT_TYPE, +} from '../../../../../../common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; + +import { INSTALL_STATES } from '../../../../../../common/types'; + +import { auditLoggingService } from '../../../../audit_logging'; +import { restartInstallation, createInstallation } from '../../install'; +import type { Installation } from '../../../../../../common'; + +import { stepCreateRestartInstallation } from './step_create_restart_installation'; + +jest.mock('../../../../audit_logging'); +jest.mock('../../install'); + +const mockedRestartInstallation = jest.mocked(restartInstallation); +const mockedCreateInstallation = createInstallation as jest.Mocked; + +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +describe('stepCreateRestartInstallation', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(() => { + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + soClient.update.mockReset(); + // mockedCreateInstallation.mockReset(); + }); + + it('Should call createInstallation if no installedPkg is available', async () => { + await stepCreateRestartInstallation({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + expect(logger.debug).toHaveBeenCalledWith(`Package install - Create installation`); + expect(mockedCreateInstallation).toHaveBeenCalledTimes(1); + }); + + it('Should call restartInstallation if installedPkg is available and force = true', async () => { + await stepCreateRestartInstallation({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + force: true, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + expect(mockedRestartInstallation).toHaveBeenCalledTimes(1); + }); + + it('Should call restartInstallation and throw if installedPkg is available and force is not provided', async () => { + const promise = stepCreateRestartInstallation({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + await expect(promise).rejects.toThrowError( + 'Concurrent installation or upgrade of xyz-4.5.6 detected, aborting.' + ); + }); + expect(mockedRestartInstallation).toHaveBeenCalledTimes(0); +}); From d49a317ac529636dd2e61dbf91ba5686d942241c Mon Sep 17 00:00:00 2001 From: criamico Date: Thu, 4 Apr 2024 22:23:10 +0200 Subject: [PATCH 34/38] Fix unit test --- .../state_machine.test.ts | 42 +++---------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts index 493021a76f16c..f6e1f8fba5a20 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts @@ -345,23 +345,9 @@ describe('handleState', () => { mockOnTransitionState3, contextData ); - try { - const updatedContext = await handleState('state1', testDefinition, testDefinition.context); - expect(updatedContext).toEqual( - expect.objectContaining({ - fixedVal: 'something', - latestExecutedState: { - name: 'state3', - started_at: expect.anything(), - error: `Error during execution of state "state1" with status "failed": Installation failed`, - }, - }) - ); - } catch (err) { - expect(err).toEqual( - `Error during execution of state "state1" with status "failed": Installation failed` - ); - } + const promise = handleState('state1', testDefinition, testDefinition.context); + await expect(promise).rejects.toThrowError('Installation failed'); + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); expect(mockOnTransitionState2).toHaveBeenCalledTimes(0); expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); @@ -447,25 +433,9 @@ describe('handleState', () => { contextData, mockPostTransition ); - try { - const updatedContext = await handleState('state1', testDefinition, testDefinition.context); - expect(updatedContext).toEqual( - expect.objectContaining({ - testData: 'test', - result1: 'test', - latestExecutedState: { - name: 'state1', - started_at: expect.anything(), - error: - 'Error during execution of state "state2" with status "failed": Installation failed', - }, - }) - ); - } catch (err) { - expect(err).toEqual( - `Error during execution of state \"state2\" with status \"failed\": Installation failed` - ); - } + const promise = handleState('state1', testDefinition, testDefinition.context); + await expect(promise).rejects.toThrowError('Installation failed'); + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); expect(mockPostTransition).toHaveBeenCalledWith( expect.objectContaining({ From 783ca850ea8da5726a8558a05068a0da4cf447eb Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 5 Apr 2024 14:31:22 +0200 Subject: [PATCH 35/38] Add more unit tests and try fixing mapping change --- .../src/constants.ts | 2 +- .../_state_machine_package_install.test.ts | 1 - .../steps/step_install_ilm_policies.test.ts | 374 +++++++++++ .../steps/step_install_ilm_policies.ts | 26 +- ...p_install_index_template_pipelines.test.ts | 592 ++++++++++++++++++ .../step_install_index_template_pipelines.ts | 16 +- .../steps/step_install_kibana_assets.test.ts | 111 ++++ 7 files changed, 1091 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts index b001229dd232d..a0ad58d5c1777 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts @@ -159,7 +159,7 @@ export const HASH_TO_VERSION_MAP = { 'endpoint:user-artifact-manifest|7502b5c5bc923abe8aa5ccfd636e8c3d': '10.0.0', 'enterprise_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', 'epm-packages-assets|44621b2f6052ef966da47b7c3a00f33b': '10.0.0', - 'epm-packages|c1e2020399dbebba2448096ca007c668': '10.1.0', + 'epm-packages|8ce219acd0f6f3529237d52193866afb': '10.2.0', 'event_loop_delays_daily|5df7e292ddd5028e07c1482e130e6654': '10.0.0', 'event-annotation-group|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0', 'exception-list-agnostic|8a1defe5981db16792cb9a772e84bb9a': '10.0.0', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts index 2d4db5ceb2f23..c77433774a5cf 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts @@ -107,7 +107,6 @@ describe('_stateMachineInstallPackage', () => { esReferences: [], }); - // use this workaround to test the error; toThrow/toThrowError doesn't match correctly const installationPromise = _stateMachineInstallPackage({ savedObjectsClient: soClient, // @ts-ignore diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts new file mode 100644 index 0000000000000..210a6b882ceed --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts @@ -0,0 +1,374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; + +import type { Installation } from '../../../../../../common'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installILMPolicy } from '../../../elasticsearch/ilm/install'; +import { installIlmForDataStream } from '../../../elasticsearch/datastream_ilm/install'; +import { ElasticsearchAssetType } from '../../../../../types'; + +jest.mock('../../../archive/storage'); +jest.mock('../../../elasticsearch/ilm/install'); +jest.mock('../../../elasticsearch/datastream_ilm/install'); + +import { stepInstallILMPolicies } from './step_install_ilm_policies'; + +describe('stepInstallILMPolicies', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(installILMPolicy).mockReset(); + jest.mocked(installIlmForDataStream).mockReset(); + }); + + it('Should not install ILM policies if disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).not.toBeCalled(); + expect(installIlmForDataStream).not.toBeCalled(); + }); + + it('Should not install ILM policies if disabled in config and should return esReferences form installedPkg', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const res = await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + installed_es: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).not.toBeCalled(); + expect(installIlmForDataStream).not.toBeCalled(); + expect(res?.esReferences).toEqual([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ]); + }); + + it('Should installs ILM policies if not disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + jest.mocked(installILMPolicy).mockResolvedValue([]); + jest.mocked(installIlmForDataStream).mockResolvedValue({ + esReferences: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + installedIlms: [], + }); + const res = await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).toHaveBeenCalled(); + expect(installIlmForDataStream).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + expect.any(Object), + expect.any(Object), + [] + ); + expect(res?.esReferences).toEqual([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ]); + }); + + it('should return updated esReferences', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + jest.mocked(installILMPolicy).mockResolvedValue([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ] as any); + jest.mocked(installIlmForDataStream).mockResolvedValue({ + esReferences: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ], + installedIlms: [], + }); + + const res = await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + expect.any(Object), + expect.any(Object), + [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ] + ); + expect(installIlmForDataStream).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + expect.any(Object), + expect.any(Object), + [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ] + ); + expect(res?.esReferences).toEqual([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts index 05ae59c712a8f..0e0d4ca2779f2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts @@ -5,8 +5,6 @@ * 2.0. */ -import type { EsAssetReference } from '../../../../../types'; - import { appContextService } from '../../../..'; import { installIlmForDataStream } from '../../../elasticsearch/datastream_ilm/install'; @@ -21,9 +19,7 @@ export async function stepInstallILMPolicies(context: InstallContext) { // Array that gets updated by each operation. This allows each operation to accurately update the // installation object with its references without requiring a refresh of the SO index on each update (faster). - const esReferences = installedPkg?.attributes.installed_es ?? []; - - let updatedEsReferences: EsAssetReference[] = []; + let esReferences = installedPkg?.attributes.installed_es ?? []; // currently only the base package has an ILM policy // at some point ILM policies can be installed/modified @@ -31,25 +27,19 @@ export async function stepInstallILMPolicies(context: InstallContext) { const isILMPoliciesDisabled = appContextService.getConfig()?.internal?.disableILMPolicies ?? false; if (!isILMPoliciesDisabled) { - updatedEsReferences = await withPackageSpan('Install ILM policies', () => - installILMPolicy( - packageInstallContext, - esClient, - savedObjectsClient, - logger, - esReferences || [] - ) + esReferences = await withPackageSpan('Install ILM policies', () => + installILMPolicy(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) ); - - const res = await withPackageSpan('Install Data Stream ILM policies', () => + ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () => installIlmForDataStream( packageInstallContext, esClient, savedObjectsClient, logger, - updatedEsReferences + esReferences ) - ); - return { esReferences: res?.esReferences || esReferences }; + )); } + // always return esReferences even when isILMPoliciesDisabled is false as it's the first time we are writing to it + return { esReferences }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts new file mode 100644 index 0000000000000..92a76eada06ec --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts @@ -0,0 +1,592 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installIndexTemplatesAndPipelines } from '../../install_index_template_pipeline'; + +jest.mock('../../install_index_template_pipeline'); + +import { stepInstallIndexTemplatePipelines } from './step_install_index_template_pipelines'; +const mockedInstallIndexTemplatesAndPipelines = + installIndexTemplatesAndPipelines as jest.MockedFunction< + typeof installIndexTemplatesAndPipelines + >; + +describe('stepInstallIndexTemplatePipelines', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedInstallIndexTemplatesAndPipelines).mockReset(); + }); + + it('Should call installIndexTemplatesAndPipelines if packageInfo type is integration', async () => { + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + const res = await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedInstallIndexTemplatesAndPipelines).toHaveBeenCalledWith({ + installedPkg: installedPkg.attributes, + packageInstallContext: expect.any(Object), + esClient: expect.any(Object), + savedObjectsClient: expect.any(Object), + logger: expect.any(Object), + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(res).toEqual({ + indexTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('Should call installIndexTemplatesAndPipelines if packageInfo type is input and installedPkg exists', async () => { + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'input', + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + policy_templates: [ + { + name: 'template_0001', + title: 'Template 1', + description: 'Template 1', + inputs: [ + { + type: 'logs', + title: 'Log', + description: 'Log Input', + vars: [ + { + name: 'path', + type: 'text', + }, + { + name: 'path_2', + type: 'text', + }, + ], + }, + ], + }, + ], + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'type-template_0001', + type: ElasticsearchAssetType.indexTemplate, + }, + ]); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + const res = await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(res).toEqual({ + indexTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('Should not call installIndexTemplatesAndPipelines if packageInfo type is input and no data streams are found', async () => { + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'input', + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + policy_templates: [ + { + name: 'template_0001', + title: 'Template 1', + description: 'Template 1', + inputs: [ + { + type: 'logs', + title: 'Log', + description: 'Log Input', + vars: [ + { + name: 'path', + type: 'text', + }, + { + name: 'path_2', + type: 'text', + }, + ], + }, + ], + }, + ], + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallIndexTemplatesAndPipelines).not.toBeCalled(); + }); + + it('Should not call installIndexTemplatesAndPipelines if packageInfo type is input and installedPkg does not exist', async () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'input', + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + policy_templates: [ + { + name: 'template_0001', + title: 'Template 1', + description: 'Template 1', + inputs: [ + { + type: 'logs', + title: 'Log', + description: 'Log Input', + vars: [ + { + name: 'path', + type: 'text', + }, + { + name: 'path_2', + type: 'text', + }, + ], + }, + ], + }, + ], + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + + await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallIndexTemplatesAndPipelines).not.toBeCalled(); + }); + + it('Should not call installIndexTemplatesAndPipelines if packageInfo type is undefined', async () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: undefined, + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + + await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + expect(mockedInstallIndexTemplatesAndPipelines).not.toBeCalled(); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts index 4eff81c691ed9..e2b6918b722cf 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts @@ -12,15 +12,9 @@ import { installIndexTemplatesAndPipelines } from '../../install_index_template_ import type { InstallContext } from '../_state_machine_package_install'; export async function stepInstallIndexTemplatePipelines(context: InstallContext) { - const { - esClient, - savedObjectsClient, - packageInstallContext, - logger, - installedPkg, - esReferences, - } = context; + const { esClient, savedObjectsClient, packageInstallContext, logger, installedPkg } = context; const { packageInfo } = packageInstallContext; + const esReferences = context.esReferences ?? []; if (packageInfo.type === 'integration') { logger.debug( @@ -33,10 +27,10 @@ export async function stepInstallIndexTemplatePipelines(context: InstallContext) esClient, savedObjectsClient, logger, - esReferences: esReferences || [], + esReferences, }); return { - esReferences: templateEsReferences || esReferences, + esReferences: templateEsReferences, indexTemplates: installedTemplates, }; } @@ -63,7 +57,7 @@ export async function stepInstallIndexTemplatePipelines(context: InstallContext) esClient, savedObjectsClient, logger, - esReferences: esReferences || [], + esReferences, onlyForDataStreams: dataStreams, }); return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts new file mode 100644 index 0000000000000..e13e3c9b095b2 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installKibanaAssetsAndReferences } from '../../../kibana/assets/install'; + +jest.mock('../../../kibana/assets/install'); + +import { stepInstallKibanaAssets } from './step_install_kibana_assets'; + +const mockedInstallKibanaAssetsAndReferences = + installKibanaAssetsAndReferences as jest.MockedFunction; + +describe('stepInstallKibanaAssets', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + + soClient.update.mockImplementation(async (type, id, attributes) => { + return { id, attributes } as any; + }); + soClient.get.mockImplementation(async (type, id) => { + return { id, attributes: {} } as any; + }); + }); + + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + + it('Should call installKibanaAssetsAndReferences', async () => { + const installationPromise = stepInstallKibanaAssets({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + await expect(installationPromise).resolves.not.toThrowError(); + expect(mockedInstallKibanaAssetsAndReferences).toBeCalledTimes(1); + }); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + + it('Should correctly handle errors', async () => { + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockImplementation(async () => { + throw new Error('mocked async error A: should be caught'); + }); + + const installationPromise = stepInstallKibanaAssets({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + await expect(installationPromise).resolves.not.toThrowError(); + await expect(installationPromise).resolves.not.toThrowError(); + }); +}); From 3a26fb473e4e87f7724e471a725600f29847dad9 Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 5 Apr 2024 15:43:19 +0200 Subject: [PATCH 36/38] Add unit test for stepDeletePreviousPipelines --- .../step_delete_previous_pipelines.test.ts | 481 ++++++++++++++++++ .../steps/step_delete_previous_pipelines.ts | 2 +- 2 files changed, 482 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts new file mode 100644 index 0000000000000..7d8a251433bb5 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts @@ -0,0 +1,481 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { + isTopLevelPipeline, + deletePreviousPipelines, +} from '../../../elasticsearch/ingest_pipeline'; + +import { stepDeletePreviousPipelines } from './step_delete_previous_pipelines'; + +jest.mock('../../../elasticsearch/ingest_pipeline'); + +const mockedDeletePreviousPipelines = deletePreviousPipelines as jest.MockedFunction< + typeof deletePreviousPipelines +>; +const mockedIsTopLevelPipeline = isTopLevelPipeline as jest.MockedFunction< + typeof isTopLevelPipeline +>; + +describe('stepDeletePreviousPipelines', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedDeletePreviousPipelines).mockReset(); + jest.mocked(mockedIsTopLevelPipeline).mockReset(); + }); + + describe('Should call deletePreviousPipelines', () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + beforeEach(async () => { + jest.mocked(mockedDeletePreviousPipelines).mockResolvedValue([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); + + it('if installType is update', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + installedPkg.attributes.name, + installedPkg.attributes.version, + [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ] + ); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is reupdate', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'reupdate', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + installedPkg.attributes.name, + installedPkg.attributes.version, + [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ] + ); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is rollback', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'rollback', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + installedPkg.attributes.name, + installedPkg.attributes.version, + [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ] + ); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + }); + + describe('Should not call deletePreviousPipelines', () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + beforeEach(async () => { + jest.mocked(mockedDeletePreviousPipelines).mockResolvedValue([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); + + it('if installType is update and installedPkg is not present', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is reupdate and installedPkg is not present', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'reupdate', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is rollback and installedPkg is not present', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'rollback', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType type is of different type', async () => { + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + jest.mocked(mockedIsTopLevelPipeline).mockImplementation(() => true); + + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { ...packageInstallContext, paths: ['some/path/1', 'some/path/2'] }, + installType: 'install', + installedPkg, + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installedPkg is present and there is a top level pipeline', async () => { + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + jest.mocked(mockedIsTopLevelPipeline).mockImplementation(() => true); + + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { ...packageInstallContext, paths: ['some/path/1', 'some/path/2'] }, + installType: 'update', + installedPkg, + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts index dc8957c871051..eb80ef16dbcb0 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts @@ -58,7 +58,7 @@ export async function stepDeletePreviousPipelines(context: InstallContext) { ) ); } else { - // if none of the previous cases, return the original esRefences + // if none of the previous cases, return the original esReferences updatedESReferences = esReferences; } return { esReferences: updatedESReferences }; From 179db75c4ad1aa45de8217d6c010d9177afe3dd8 Mon Sep 17 00:00:00 2001 From: criamico Date: Fri, 5 Apr 2024 17:46:17 +0200 Subject: [PATCH 37/38] More unit tests for install steps --- .../steps/step_install_mlmodel.test.ts | 155 +++++++++++++++ .../steps/step_install_mlmodel.ts | 9 +- .../steps/step_install_transforms.test.ts | 161 +++++++++++++++ .../steps/step_install_transforms.ts | 8 +- .../step_remove_legacy_templates.test.ts | 155 +++++++++++++++ .../steps/step_save_archive_entries.test.ts | 184 ++++++++++++++++++ .../steps/step_save_archive_entries.ts | 4 +- .../step_update_current_write_indices.test.ts | 163 ++++++++++++++++ 8 files changed, 829 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts new file mode 100644 index 0000000000000..ac67f8abfaccb --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installMlModel } from '../../../elasticsearch/ml_model'; + +import { stepInstallMlModel } from './step_install_mlmodel'; + +jest.mock('../../../elasticsearch/ml_model'); + +const mockedInstallMlModel = installMlModel as jest.MockedFunction; + +describe('stepInstallMlModel', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedInstallMlModel).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should update esReferences', async () => { + jest.mocked(mockedInstallMlModel).mockResolvedValue([]); + const res = await stepInstallMlModel({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallMlModel).toHaveBeenCalled(); + expect(res.esReferences).toEqual([]); + }); + + it('Should call installTransforms and return updated esReferences', async () => { + jest.mocked(mockedInstallMlModel).mockResolvedValue([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + const res = await stepInstallMlModel({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + expect(mockedInstallMlModel).toHaveBeenCalled(); + expect(res.esReferences).toEqual([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts index a121d543f8d7a..31d571fee4505 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts @@ -12,10 +12,11 @@ import { withPackageSpan } from '../../utils'; import type { InstallContext } from '../_state_machine_package_install'; export async function stepInstallMlModel(context: InstallContext) { - const { logger, esReferences, packageInstallContext, esClient, savedObjectsClient } = context; + const { logger, packageInstallContext, esClient, savedObjectsClient } = context; + let esReferences = context.esReferences ?? []; - const updatedEsReferences = await withPackageSpan('Install ML models', () => - installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences || []) + esReferences = await withPackageSpan('Install ML models', () => + installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) ); - return { esReferences: updatedEsReferences || esReferences }; + return { esReferences }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts new file mode 100644 index 0000000000000..63ea9c203bf43 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installTransforms } from '../../../elasticsearch/transform/install'; + +import { stepInstallTransforms } from './step_install_transforms'; + +jest.mock('../../../elasticsearch/transform/install'); + +const mockedInstallTransforms = installTransforms as jest.MockedFunction; + +describe('stepInstallTransforms', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedInstallTransforms).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should update esReferences', async () => { + jest.mocked(mockedInstallTransforms).mockResolvedValue({ + installedTransforms: [], + esReferences: [], + }); + const res = await stepInstallTransforms({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallTransforms).toHaveBeenCalled(); + expect(res.esReferences).toEqual([]); + }); + + it('Should call installTransforms and return updated esReferences', async () => { + jest.mocked(mockedInstallTransforms).mockResolvedValue({ + installedTransforms: [], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const res = await stepInstallTransforms({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + expect(mockedInstallTransforms).toHaveBeenCalled(); + expect(res.esReferences).toEqual([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts index e04f831dd28d2..cd7d7404db5ad 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts @@ -17,12 +17,12 @@ export async function stepInstallTransforms(context: InstallContext) { esClient, savedObjectsClient, logger, - esReferences, force, authorizationHeader, } = context; + let esReferences = context.esReferences ?? []; - const res = await withPackageSpan('Install transforms', () => + ({ esReferences } = await withPackageSpan('Install transforms', () => installTransforms({ packageInstallContext, esClient, @@ -32,6 +32,6 @@ export async function stepInstallTransforms(context: InstallContext) { force, authorizationHeader, }) - ); - return { esReferences: res.esReferences || esReferences }; + )); + return { esReferences }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts new file mode 100644 index 0000000000000..39e7159596ba8 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { removeLegacyTemplates } from '../../../elasticsearch/template/remove_legacy'; + +import { stepRemoveLegacyTemplates } from './step_remove_legacy_templates'; + +jest.mock('../../../elasticsearch/template/remove_legacy'); + +const mockedRemoveLegacyTemplates = removeLegacyTemplates as jest.MockedFunction< + typeof removeLegacyTemplates +>; + +describe('stepRemoveLegacyTemplates', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedRemoveLegacyTemplates).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should call removeLegacyTemplates', async () => { + await stepRemoveLegacyTemplates({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedRemoveLegacyTemplates).toHaveBeenCalled(); + }); + + it('Should catch the error when removeLegacyTemplates fails', async () => { + jest.mocked(mockedRemoveLegacyTemplates).mockRejectedValue(Error('Error!')); + await stepRemoveLegacyTemplates({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedRemoveLegacyTemplates).toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith('Error removing legacy templates: Error!'); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts new file mode 100644 index 0000000000000..3515fd304b356 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { saveArchiveEntriesFromAssetsMap } from '../../../archive/storage'; + +import { stepSaveArchiveEntries } from './step_save_archive_entries'; + +jest.mock('../../../archive/storage'); + +const mockedSaveArchiveEntriesFromAssetsMap = + saveArchiveEntriesFromAssetsMap as jest.MockedFunction; + +describe('stepSaveArchiveEntries', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedSaveArchiveEntriesFromAssetsMap).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', + Buffer.from('{"content": "data"}'), + ], + ]), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should return empty packageAssetRefs if saved_objects were not found', async () => { + jest.mocked(mockedSaveArchiveEntriesFromAssetsMap).mockResolvedValue({ + saved_objects: [], + }); + const res = await stepSaveArchiveEntries({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(res).toEqual({ + packageAssetRefs: [], + }); + }); + + it('Should return packageAssetRefs', async () => { + jest.mocked(mockedSaveArchiveEntriesFromAssetsMap).mockResolvedValue({ + saved_objects: [ + { + id: 'test', + attributes: { + package_name: 'test-package', + package_version: '1.0.0', + install_source: 'registry', + asset_path: 'some/path', + media_type: '', + data_utf8: '', + data_base64: '', + }, + type: '', + references: [], + }, + ], + }); + const res = await stepSaveArchiveEntries({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(res).toEqual({ + packageAssetRefs: [ + { + id: 'test', + type: 'epm-packages-assets', + }, + ], + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index 3e4702c459eed..ca65b04e55303 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -22,8 +22,8 @@ export async function stepSaveArchiveEntries(context: InstallContext) { const packageAssetResults = await withPackageSpan('Update archive entries', () => saveArchiveEntriesFromAssetsMap({ savedObjectsClient, - assetsMap: packageInstallContext.assetsMap, - paths: packageInstallContext.paths, + assetsMap: packageInstallContext?.assetsMap, + paths: packageInstallContext?.paths, packageInfo, installSource, }) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts new file mode 100644 index 0000000000000..c7f3c040b7966 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import type { IndicesGetIndexTemplateIndexTemplateItem } from '@elastic/elasticsearch/lib/api/types'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { updateCurrentWriteIndices } from '../../../elasticsearch/template/template'; + +import { stepUpdateCurrentWriteIndices } from './step_update_current_write_indices'; + +jest.mock('../../../elasticsearch/template/template'); + +const mockedUpdateCurrentWriteIndices = updateCurrentWriteIndices as jest.MockedFunction< + typeof updateCurrentWriteIndices +>; + +const createMockTemplate = ({ name, composedOf = [] }: { name: string; composedOf?: string[] }) => + ({ + name, + index_template: { + composed_of: composedOf, + }, + } as IndicesGetIndexTemplateIndexTemplateItem); + +describe('stepUpdateCurrentWriteIndices', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedUpdateCurrentWriteIndices).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should call updateCurrentWriteIndices', async () => { + await stepUpdateCurrentWriteIndices({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + + expect(mockedUpdateCurrentWriteIndices).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + [], + { ignoreMappingUpdateErrors: undefined, skipDataStreamRollover: undefined } + ); + }); + + it('Should call updateCurrentWriteIndices with passed parameters', async () => { + const indexTemplates = [createMockTemplate({ name: 'tmpl1' })] as any; + await stepUpdateCurrentWriteIndices({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + indexTemplates, + ignoreMappingUpdateErrors: true, + skipDataStreamRollover: true, + }); + + expect(mockedUpdateCurrentWriteIndices).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + indexTemplates, + { ignoreMappingUpdateErrors: true, skipDataStreamRollover: true } + ); + }); +}); From 4a96479920df90f4f6d377efce02202742362fba Mon Sep 17 00:00:00 2001 From: criamico Date: Mon, 8 Apr 2024 10:38:14 +0200 Subject: [PATCH 38/38] Add unit test for stepSaveSystemObject --- .../steps/step_save_system_object.test.ts | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts new file mode 100644 index 0000000000000..e91826c99793c --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; + +import { auditLoggingService } from '../../../../audit_logging'; +import { packagePolicyService } from '../../../../package_policy'; + +import { stepSaveSystemObject } from './step_save_system_object'; + +jest.mock('../../../../audit_logging'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +jest.mock('../../../../package_policy'); +const mockedPackagePolicyService = packagePolicyService as jest.Mocked; + +describe('updateLatestExecutedState', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + + afterEach(() => { + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + soClient.get.mockReset(); + soClient.update.mockReset(); + }); + + it('Should save the SO and should not call packagePolicy upgrade if keep_policies_up_to_date = false', async () => { + soClient.get.mockResolvedValue({ + id: 'test-integration', + attributes: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + install_source: 'registry', + install_status: 'installed', + package_assets: [], + }, + } as any); + + await stepSaveSystemObject({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'test-integration', + { + install_format_schema_version: '1.2.0', + install_status: 'installed', + install_version: '1.0.0', + latest_install_failed_attempts: [], + package_assets: undefined, + version: '1.0.0', + }, + ], + ]); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'test-integration', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + expect(mockedPackagePolicyService.upgrade).not.toBeCalled(); + }); + + it('Should call packagePolicy upgrade if keep_policies_up_to_date = true', async () => { + soClient.get.mockResolvedValue({ + id: 'test-integration', + attributes: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + install_source: 'registry', + install_status: 'installed', + package_assets: [], + keep_policies_up_to_date: true, + }, + } as any); + mockedPackagePolicyService.listIds.mockReturnValue({ + items: ['packagePolicy1', 'packagePolicy2'], + } as any); + + await stepSaveSystemObject({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'test-integration', + { + install_format_schema_version: '1.2.0', + install_status: 'installed', + install_version: '1.0.0', + latest_install_failed_attempts: [], + package_assets: undefined, + version: '1.0.0', + }, + ], + ]); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'test-integration', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + expect(packagePolicyService.upgrade).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + ['packagePolicy1', 'packagePolicy2'] + ); + }); +});