[Fleet] Implement state machine behavior for package install (#178657)

Closes #175592 ## Summary Implement state machine behavior for package install. It keeps track of the current step and save it in the SO , then exposes it in the`installationInfo` property. - Implemented a generic state machine function that can automatically handle state transitions based on a simple data structure: https://github.com/elastic/kibana/pull/178657/files#diff-f350d9630cd1f22cd1b3e70c9e95388d72dc877190bbeb33c739cb0433949e95R1-R88. In theory, this state machine could be reused for something else, since is generic enough and it's decoupled from the transition functions that we pass to it. - The state transitions passed to the state machine are defined in [services/epm/packages/install_steps.ts](https://github.com/elastic/kibana/blob/5f09e58ae7a300f459c3d1157fb747cfbb0c11aa/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts) and are based off the existing steps in https://github.com/elastic/kibana/blob/10d5167fa78c1a4c65f8607dad1e6a681e39f4b0/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts#L61 I simply divided that long function in smaller steps and wrapped them to accept a common parameter, based off [InstallContext](https://github.com/elastic/kibana/pull/178657/files#diff-39d1f59e77a329eb06241c220156e5cf2d350649bb548707b0b0f54365ea91bfR49-R72) - Defined a **feature flag** `enablePackagesStateMachine` and called the new [installPackageWitStateMachine](https://github.com/elastic/kibana/pull/178657/files#diff-cf9cec44de2ad0a6a3b74cca05e5308231d57d5c3e180ae4bc5114c2bf9af4ebR466-R483) only when it's enabled. - For now this new function is only applied to `InstallPackageFromRegistry`, so `upload` and `bundled` case don't use it yet. ### Testing - Enable `enablePackagesStateMachine` in kibana.dev.yml - Try to install an integration from registry, either from API or UI. For instance ``` POST kbn:api/fleet/epm/packages/nginx/1.20.0 ``` The installation process should succeed and the installationInfo property will expose the `latest_executed_state` along with the error. <details> <summary>Screenshots</summary> ### Logging With `logger.debug` enabled: ![Screenshot 2024-03-27 at 16 12 33](https://github.com/elastic/kibana/assets/16084106/75fb4af8-675e-483e-a51f-eb4adbf9d2aa) ![Screenshot 2024-03-27 at 16 12 48](https://github.com/elastic/kibana/assets/16084106/74092f6d-528c-4e8f-85ee-85e2852487b8) ### InstallationInfo object Content of `installationInfo` property when install process was successful: ![Screenshot 2024-03-27 at 16 13 54](https://github.com/elastic/kibana/assets/16084106/c2535c8f-24f7-4b6c-8f58-dadf4c9b4b28) ### Errors during install process I manually triggered an error inside `stepInstallIndexTemplatePipelines` and it's reported in the `installationInfo` property along with the latest executed step (latest successful state) and error message: ![Screenshot 2024-03-27 at 17 26 29](https://github.com/elastic/kibana/assets/16084106/47d77330-bcbb-4608-9e42-c9f46e8831a1) </details> ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Apr 8, 2024 · 3a31ee0 · 3a31ee0
1 parent 9f8433e
commit 3a31ee0
Show file tree

Hide file tree

Showing 51 changed files with 5,672 additions and 216 deletions.
diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts
@@ -159,7 +159,7 @@ export const HASH_TO_VERSION_MAP = {
   'endpoint:user-artifact-manifest|7502b5c5bc923abe8aa5ccfd636e8c3d': '10.0.0',
   'enterprise_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
   'epm-packages-assets|44621b2f6052ef966da47b7c3a00f33b': '10.0.0',
-  'epm-packages|c1e2020399dbebba2448096ca007c668': '10.1.0',
+  'epm-packages|8ce219acd0f6f3529237d52193866afb': '10.2.0',
   'event_loop_delays_daily|5df7e292ddd5028e07c1482e130e6654': '10.0.0',
   'event-annotation-group|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0',
   'exception-list-agnostic|8a1defe5981db16792cb9a772e84bb9a': '10.0.0',

diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json
@@ -290,6 +290,7 @@
     "installed_kibana_space_id",
     "internal",
     "keep_policies_up_to_date",
+    "latest_executed_state",
     "latest_install_failed_attempts",
     "name",
     "package_assets",

diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1003,6 +1003,10 @@
         "index": false,
         "type": "boolean"
       },
+      "latest_executed_state": {
+        "enabled": false,
+        "type": "object"
+      },
       "latest_install_failed_attempts": {
         "enabled": false,
         "type": "object"

diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -85,7 +85,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "dashboard": "211e9ca30f5a95d5f3c27b1bf2b58e6cfa0c9ae9",
         "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
-        "epm-packages": "c23d3d00c051a08817335dba26f542b64b18a56a",
+        "epm-packages": "f8ee125b57df31fd035dc04ad81aef475fd2f5bd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",
         "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88",

diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts
@@ -28,6 +28,7 @@ export const allowedExperimentalValues = Object.freeze<Record<string, boolean>>(
   agentless: false,
   enableStrictKQLValidation: false,
   subfeaturePrivileges: false,
+  enablePackagesStateMachine: true,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json
@@ -6302,6 +6302,34 @@
               }
             }
           },
+          "latest_executed_state": {
+            "description": "Latest successfully executed state in package install state machine",
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "enum": [
+                  "create_restart_installation",
+                  "install_kibana_assets",
+                  "install_ilm_policies",
+                  "install_ml_model",
+                  "install_index_template_pipelines",
+                  "remove_legacy_templates",
+                  "update_current_write_indices",
+                  "install_transforms",
+                  "delete_previous_pipelines",
+                  "save_archive_entries_from_assets_map",
+                  "update_so"
+                ]
+              },
+              "started_at": {
+                "type": "string"
+              },
+              "error": {
+                "type": "string"
+              }
+            }
+          },
           "verification_status": {
             "type": "string",
             "enum": [

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml
@@ -3970,6 +3970,28 @@ components:
                     type: string
                   stack:
                     type: string
+        latest_executed_state:
+          description: Latest successfully executed state in package install state machine
+          type: object
+          properties:
+            name:
+              type: string
+              enum:
+                - create_restart_installation
+                - install_kibana_assets
+                - install_ilm_policies
+                - install_ml_model
+                - install_index_template_pipelines
+                - remove_legacy_templates
+                - update_current_write_indices
+                - install_transforms
+                - delete_previous_pipelines
+                - save_archive_entries_from_assets_map
+                - update_so
+            started_at:
+              type: string
+            error:
+              type: string
         verification_status:
           type: string
           enum:

diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml
@@ -66,6 +66,28 @@ properties:
               type: string
             stack:
               type: string
+  latest_executed_state:
+    description: Latest successfully executed state in package install state machine
+    type: object
+    properties:
+      name:
+        type: string
+        enum:
+          - create_restart_installation
+          - install_kibana_assets
+          - install_ilm_policies
+          - install_ml_model
+          - install_index_template_pipelines
+          - remove_legacy_templates
+          - update_current_write_indices
+          - install_transforms
+          - delete_previous_pipelines
+          - save_archive_entries_from_assets_map
+          - update_so
+      started_at:
+        type: string
+      error:
+        type: string
   verification_status:
     type: string
     enum:

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -37,6 +37,7 @@ export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'in
 export type InstallSource = 'registry' | 'upload' | 'bundled' | 'custom';
 
 export type EpmPackageInstallStatus = 'installed' | 'installing' | 'install_failed';
+export type InstallResultStatus = 'installed' | 'already_installed';
 
 export type ServiceName = 'kibana' | 'elasticsearch';
 export type AgentAssetType = typeof agentAssetTypes;
@@ -548,6 +549,36 @@ export interface InstallFailedAttempt {
   };
 }
 
+export enum INSTALL_STATES {
+  CREATE_RESTART_INSTALLATION = 'create_restart_installation',
+  INSTALL_KIBANA_ASSETS = 'install_kibana_assets',
+  INSTALL_ILM_POLICIES = 'install_ilm_policies',
+  INSTALL_ML_MODEL = 'install_ml_model',
+  INSTALL_INDEX_TEMPLATE_PIPELINES = 'install_index_template_pipelines',
+  REMOVE_LEGACY_TEMPLATES = 'remove_legacy_templates',
+  UPDATE_CURRENT_WRITE_INDICES = 'update_current_write_indices',
+  INSTALL_TRANSFORMS = 'install_transforms',
+  DELETE_PREVIOUS_PIPELINES = 'delete_previous_pipelines',
+  SAVE_ARCHIVE_ENTRIES = 'save_archive_entries_from_assets_map',
+  RESOLVE_KIBANA_PROMISE = 'resolve_kibana_promise',
+  UPDATE_SO = 'update_so',
+}
+type StatesKeys = keyof typeof INSTALL_STATES;
+export type StateNames = typeof INSTALL_STATES[StatesKeys];
+
+export interface LatestExecutedState<T> {
+  name: T;
+  started_at: string;
+  error?: string;
+}
+
+export type InstallLatestExecutedState = LatestExecutedState<StateNames>;
+
+export interface StateContext<T> {
+  [key: string]: any;
+  latestExecutedState?: LatestExecutedState<T>;
+}
+
 export interface Installation {
   installed_kibana: KibanaAssetReference[];
   installed_es: EsAssetReference[];
@@ -568,6 +599,7 @@ export interface Installation {
   internal?: boolean;
   removable?: boolean;
   latest_install_failed_attempts?: InstallFailedAttempt[];
+  latest_executed_state?: InstallLatestExecutedState;
 }
 
 export interface PackageUsageStats {

diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts
@@ -18,6 +18,7 @@ import type {
   EpmPackageInstallStatus,
   SimpleSOAssetType,
   AssetSOObject,
+  InstallResultStatus,
 } from '../models/epm';
 
 export interface GetCategoriesRequest {
@@ -154,7 +155,7 @@ export interface IBulkInstallPackageHTTPError {
 
 export interface InstallResult {
   assets?: AssetReference[];
-  status?: 'installed' | 'already_installed';
+  status?: InstallResultStatus;
   error?: Error;
   installType: InstallType;
   installSource: InstallSource;

diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts
@@ -612,6 +612,7 @@ const soToInstallationInfo = (pkg: PackageListItem | PackageInfo) => {
       verification_key_id: attributes.verification_key_id,
       experimental_data_stream_features: attributes.experimental_data_stream_features,
       latest_install_failed_attempts: attributes.latest_install_failed_attempts,
+      latest_executed_state: attributes.latest_executed_state,
     };
 
     return {

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -530,6 +530,7 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
           },
         },
         latest_install_failed_attempts: { type: 'object', enabled: false },
+        latest_executed_state: { type: 'object', enabled: false },
         installed_kibana: {
           dynamic: false,
           properties: {},
@@ -571,6 +572,16 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
           },
         ],
       },
+      '2': {
+        changes: [
+          {
+            type: 'mappings_addition',
+            addedMappings: {
+              latest_executed_state: { type: 'object', enabled: false },
+            },
+          },
+        ],
+      },
     },
     migrations: {
       '7.14.0': migrateInstallationToV7140,

diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts
@@ -172,7 +172,6 @@ export async function installKibanaAssetsAndReferences({
   pkgName,
   pkgTitle,
   packageInstallContext,
-  paths,
   installedPkg,
   spaceId,
   assetTags,
@@ -185,7 +184,6 @@ export async function installKibanaAssetsAndReferences({
   pkgName: string;
   pkgTitle: string;
   packageInstallContext: PackageInstallContext;
-  paths: string[];
   installedPkg?: SavedObject<Installation>;
   spaceId: string;
   assetTags?: PackageSpecTags[];

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
@@ -161,7 +161,6 @@ export async function _installPackage({
         pkgName,
         pkgTitle,
         packageInstallContext,
-        paths,
         installedPkg,
         logger,
         spaceId,