[Fleet] Improve performance of Fleet setup

x-pack/plugins/fleet/common/constants/epm.ts

@@ -9,17 +9,35 @@ export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;
 
+export const FLEET_SYSTEM_PACKAGE = 'system';
+export const FLEET_ELASTIC_AGENT_PACKAGE = 'elastic_agent';
 export const FLEET_SERVER_PACKAGE = 'fleet_server';
+export const FLEET_ENDPOINT_PACKAGE = 'endpoint';
 
-export const requiredPackages = {
-  System: 'system',
-  Endpoint: 'endpoint',
-  ElasticAgent: 'elastic_agent',
-  FleetServer: FLEET_SERVER_PACKAGE,
-} as const;
+/*
+ Package rules:
+|               | requiredPackages | defaultPackages | autoUpdatePackages |
+|---------------|:----------------:|:---------------:|:------------------:|
+| Removable     |         ❌        |        ✔️        |          ✔️         |
+| Auto-installs |         ❌        |        ✔️        |          ❌         |
+| Auto-updates  |         ❌        |        ✔️        |          ✔️         |
+
+`endpoint` is a special package. It needs to autoupdate, it needs to _not_ be
+removable, but it doesn't install by default. Following the table, it needs to
+be in `requiredPackages` and in `autoUpdatePackages`, but not in
+`defaultPackages`.
+*/
+
+export const requiredPackages = [
+  FLEET_SYSTEM_PACKAGE,
+  FLEET_ELASTIC_AGENT_PACKAGE,
+  FLEET_SERVER_PACKAGE,
+  FLEET_ENDPOINT_PACKAGE,
+];
+
+export const defaultPackages = requiredPackages.filter((p) => p !== FLEET_ENDPOINT_PACKAGE);
 
-// these are currently identical. we can separate if they later diverge
-export const defaultPackages = requiredPackages;
+export const autoUpdatePackages = [FLEET_ENDPOINT_PACKAGE];
 
 export const agentAssetTypes = {
   Input: 'input',

x-pack/plugins/fleet/common/constants/preconfiguration.ts

@@ -7,7 +7,13 @@
 
 import type { PreconfiguredAgentPolicy } from '../types';
 
-import { defaultPackages } from './epm';
+import {
+  defaultPackages,
+  requiredPackages,
+  FLEET_SYSTEM_PACKAGE,
+  FLEET_SERVER_PACKAGE,
+  autoUpdatePackages,
+} from './epm';
 
 export const PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE =
   'fleet-preconfiguration-deletion-record';
@@ -27,9 +33,9 @@ export const DEFAULT_AGENT_POLICY: PreconfiguredAgentPolicyWithDefaultInputs = {
   description: 'Default agent policy created by Kibana',
   package_policies: [
     {
-      name: `${defaultPackages.System}-1`,
+      name: `${FLEET_SYSTEM_PACKAGE}-1`,
       package: {
-        name: defaultPackages.System,
+        name: FLEET_SYSTEM_PACKAGE,
       },
     },
   ],
@@ -44,9 +50,9 @@ export const DEFAULT_FLEET_SERVER_AGENT_POLICY: PreconfiguredAgentPolicyWithDefa
   description: 'Default Fleet Server agent policy created by Kibana',
   package_policies: [
     {
-      name: `${defaultPackages.FleetServer}-1`,
+      name: `${FLEET_SERVER_PACKAGE}-1`,
       package: {
-        name: defaultPackages.FleetServer,
+        name: FLEET_SERVER_PACKAGE,
       },
     },
   ],
@@ -56,13 +62,20 @@ export const DEFAULT_FLEET_SERVER_AGENT_POLICY: PreconfiguredAgentPolicyWithDefa
   monitoring_enabled: ['logs', 'metrics'] as Array<'logs' | 'metrics'>,
 };
 
-export const DEFAULT_PACKAGES = Object.values(defaultPackages).map((name) => ({
+export const REQUIRED_PACKAGES = requiredPackages.map((name) => ({
   name,
   version: PRECONFIGURATION_LATEST_KEYWORD,
 }));
 
-// these are currently identical. we can separate if they later diverge
-export const REQUIRED_PACKAGES = DEFAULT_PACKAGES;
+export const DEFAULT_PACKAGES = defaultPackages.map((name) => ({
+  name,
+  version: PRECONFIGURATION_LATEST_KEYWORD,
+}));
+
+export const AUTO_UPDATE_PACKAGES = autoUpdatePackages.map((name) => ({
+  name,
+  version: PRECONFIGURATION_LATEST_KEYWORD,
+}));
 
 export interface PreconfigurationError {
   package?: { name: string; version: string };

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -13,9 +13,7 @@ import type {
   ASSETS_SAVED_OBJECT_TYPE,
   agentAssetTypes,
   dataTypes,
-  defaultPackages,
   installationStatuses,
-  requiredPackages,
 } from '../../constants';
 import type { ValueOf } from '../../types';
 
@@ -408,10 +406,6 @@ export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
   type: typeof ASSETS_SAVED_OBJECT_TYPE;
 };
 
-export type RequiredPackage = typeof requiredPackages;
-
-export type DefaultPackages = typeof defaultPackages;
-
 export interface IndexTemplateMappings {
   properties: any;
 }

x-pack/plugins/fleet/public/applications/fleet/app.tsx

@@ -35,7 +35,7 @@ import {
   useStartServices,
   UIExtensionsContext,
 } from './hooks';
-import { Error, Loading, SettingFlyout } from './components';
+import { Error, Loading, SettingFlyout, FleetSetupLoading } from './components';
 import type { UIExtensionsStorage } from './types';
 
 import { FLEET_ROUTING_PATHS } from './constants';
@@ -180,7 +180,7 @@ export const WithPermissionsAndSetup: React.FC = memo(({ children }) => {
             error={initializationError}
           />
         ) : (
-          <Loading />
+          <FleetSetupLoading />
         )}
       </ErrorLayout>
     );

x-pack/plugins/fleet/public/components/fleet_setup_loading.tsx

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui';
+
+export const FleetSetupLoading: React.FunctionComponent = () => (
+  <EuiEmptyPrompt
+    title={
+      <h2>
+        <FormattedMessage id="xpack.fleet.setup.titleLabel" defaultMessage="Loading Fleet..." />
+      </h2>
+    }
+    titleSize="m"
+    body={<EuiLoadingSpinner size="xl" />}
+  />
+);

x-pack/plugins/fleet/public/components/index.ts

@@ -6,6 +6,7 @@
  */
 export { Loading } from './loading';
 export { Error } from './error';
+export { FleetSetupLoading } from './fleet_setup_loading';
 export { PackageIcon } from './package_icon';
 export { ContextMenuActions } from './context_menu_actions';
 export { LinkedAgentCount } from './linked_agent_count';

x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts

@@ -23,7 +23,7 @@ import type {
   GetFullAgentPolicyRequestSchema,
 } from '../../types';
 import type { AgentPolicy, NewPackagePolicy } from '../../types';
-import { defaultPackages } from '../../../common';
+import { FLEET_SYSTEM_PACKAGE } from '../../../common';
 import type {
   GetAgentPoliciesResponse,
   GetAgentPoliciesResponseItem,
@@ -120,7 +120,7 @@ export const createAgentPolicyHandler: RequestHandler<
       // successfully
       withSysMonitoring
         ? packagePolicyService
-            .buildPackagePolicyFromPackage(soClient, defaultPackages.System)
+            .buildPackagePolicyFromPackage(soClient, FLEET_SYSTEM_PACKAGE)
             .catch(() => undefined)
         : undefined,
     ]);

x-pack/plugins/fleet/server/services/epm/packages/index.ts

@@ -8,7 +8,6 @@
 import type { SavedObject } from 'src/core/server';
 
 import { requiredPackages, installationStatuses } from '../../../../common';
-import type { RequiredPackage, ValueOf } from '../../../../common';
 import { KibanaAssetType } from '../../../types';
 import type { AssetType, Installable, Installation } from '../../../types';
 
@@ -33,8 +32,8 @@ export {
 } from './install';
 export { removeInstallation } from './remove';
 
-export function isRequiredPackage(value: string): value is ValueOf<RequiredPackage> {
-  return Object.values(requiredPackages).some((required) => value === required);
+export function isRequiredPackage(value: string): boolean {
+  return requiredPackages.includes(value);
 }
 
 export class PackageNotInstalledError extends Error {

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -39,6 +39,14 @@ import { removeInstallation } from './remove';
 import { getPackageSavedObjects } from './get';
 import { _installPackage } from './_install_package';
 
+export async function isPackageInstalled(options: {
+  savedObjectsClient: SavedObjectsClientContract;
+  pkgName: string;
+}): Promise<boolean> {
+  const installedPackage = await getInstallation(options);
+  return installedPackage !== undefined;
+}
+
 export async function isPackageVersionOrLaterInstalled(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;

x-pack/plugins/fleet/server/services/setup.ts

@@ -5,10 +5,13 @@
  * 2.0.
  */
 
+import { compact } from 'lodash';
+
 import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
 
+import { AUTO_UPDATE_PACKAGES } from '../../common';
 import type { DefaultPackagesInstallationError, PreconfigurationError } from '../../common';
-import { SO_SEARCH_LIMIT, REQUIRED_PACKAGES } from '../constants';
+import { SO_SEARCH_LIMIT, DEFAULT_PACKAGES } from '../constants';
 
 import { appContextService } from './app_context';
 import { agentPolicyService } from './agent_policy';
@@ -21,6 +24,7 @@ import { awaitIfPending } from './setup_utils';
 import { ensureAgentActionPolicyChangeExists } from './agents';
 import { awaitIfFleetServerSetupPending } from './fleet_server';
 import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install';
+import { isPackageInstalled } from './epm/packages/install';
 
 export interface SetupStatus {
   isInitialized: boolean;
@@ -53,11 +57,25 @@ async function createSetupSideEffects(
   const policies = policiesOrUndefined ?? [];
 
   let packages = packagesOrUndefined ?? [];
+
   // Ensure that required packages are always installed even if they're left out of the config
   const preconfiguredPackageNames = new Set(packages.map((pkg) => pkg.name));
+
+  const autoUpdateablePackages = compact(
+    await Promise.all(
+      AUTO_UPDATE_PACKAGES.map((pkg) =>
+        isPackageInstalled({
+          savedObjectsClient: soClient,
+          pkgName: pkg.name,
+        }).then((installed) => (installed ? pkg : undefined))
+      )
+    )
+  );
+
   packages = [
     ...packages,
-    ...REQUIRED_PACKAGES.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)),
+    ...DEFAULT_PACKAGES.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)),
+    ...autoUpdateablePackages.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)),
   ];
 
   const { nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies(

x-pack/plugins/fleet/server/types/index.tsx

@@ -63,7 +63,6 @@ export {
   IndexTemplate,
   RegistrySearchResults,
   RegistrySearchResult,
-  DefaultPackages,
   TemplateRef,
   IndexTemplateMappings,
   Settings,

x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts

@@ -7,11 +7,8 @@
 
 import { SavedObjectsFindResponse } from 'src/core/server';
 
-import { Agent } from '../../../../fleet/common';
-import {
-  FLEET_ENDPOINT_PACKAGE_CONSTANT,
-  AGENT_EVENT_SAVED_OBJECT_TYPE,
-} from './fleet_saved_objects';
+import { Agent, FLEET_ENDPOINT_PACKAGE } from '../../../../fleet/common';
+import { AGENT_EVENT_SAVED_OBJECT_TYPE } from './fleet_saved_objects';
 
 const testAgentId = 'testAgentId';
 const testAgentPolicyId = 'testAgentPolicyId';
@@ -65,7 +62,7 @@ export const mockFleetObjectsResponse = (
           full: MockOSFullName,
         },
       },
-      packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
+      packages: [FLEET_ENDPOINT_PACKAGE, 'system'],
       last_checkin: lastCheckIn,
     },
     {
@@ -93,7 +90,7 @@ export const mockFleetObjectsResponse = (
           full: MockOSFullName,
         },
       },
-      packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
+      packages: [FLEET_ENDPOINT_PACKAGE, 'system'],
       last_checkin: lastCheckIn,
     },
   ],

x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts

@@ -11,9 +11,7 @@ import {
   SavedObjectsFindResponse,
 } from 'src/core/server';
 import { AgentService } from '../../../../fleet/server';
-import { defaultPackages as FleetDefaultPackages } from '../../../../fleet/common';
-
-export const FLEET_ENDPOINT_PACKAGE_CONSTANT = FleetDefaultPackages.Endpoint;
+import { FLEET_ENDPOINT_PACKAGE } from '../../../../fleet/common';
 
 export const AGENT_EVENT_SAVED_OBJECT_TYPE = 'donotexistsanymore-since-7.13';
 
@@ -22,7 +20,7 @@ export const getEndpointIntegratedFleetMetadata = async (
   esClient: ElasticsearchClient
 ) => {
   return agentService?.listAgents(esClient, {
-    kuery: `(packages : ${FLEET_ENDPOINT_PACKAGE_CONSTANT})`,
+    kuery: `(packages : ${FLEET_ENDPOINT_PACKAGE})`,
     perPage: 10000,
     showInactive: false,
     sortField: 'enrolled_at',