From ebc02741b3ec5e0caaa6a811d5aa066d030202a0 Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Mon, 19 Jul 2021 18:42:11 +0200 Subject: [PATCH 01/39] [Security Solution]Memory protection configuration card for policies integration. (#101365) --- .../fleet/server/saved_objects/index.ts | 2 + .../migrations/security_solution/index.ts | 1 + .../security_solution/to_v7_15_0.test.ts | 174 +++++++++++++++++ .../security_solution/to_v7_15_0.ts | 43 +++++ .../saved_objects/migrations/to_v7_15_0.ts | 26 +++ .../common/endpoint/models/policy_config.ts | 22 ++- .../common/endpoint/types/index.ts | 13 +- .../common/license/policy_config.test.ts | 180 ++++++++++++++---- .../common/license/policy_config.ts | 44 ++++- .../policy/store/policy_details/index.test.ts | 5 + .../policy/store/policy_details/middleware.ts | 15 +- .../policy/store/policy_details/selectors.ts | 1 + .../public/management/pages/policy/types.ts | 8 +- .../pages/policy/view/policy_details.test.tsx | 21 +- .../pages/policy/view/policy_details_form.tsx | 24 ++- .../policy/view/policy_forms/locked_card.tsx | 13 +- .../view/policy_forms/protections/memory.tsx | 60 ++++++ .../protections/popup_options_to_versions.ts | 1 + .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - .../apps/endpoint/policy_details.ts | 15 ++ 21 files changed, 611 insertions(+), 59 deletions(-) create mode 100644 x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.test.ts create mode 100644 x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.ts create mode 100644 x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_15_0.ts create mode 100644 x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/memory.tsx diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index fe8771115a2174..9f9f0dab6efacd 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -43,6 +43,7 @@ import { migrateOutputToV7130, } from './migrations/to_v7_13_0'; import { migratePackagePolicyToV7140 } from './migrations/to_v7_14_0'; +import { migratePackagePolicyToV7150 } from './migrations/to_v7_15_0'; /* * Saved object types and mappings @@ -272,6 +273,7 @@ const getSavedObjectTypes = ( '7.12.0': migratePackagePolicyToV7120, '7.13.0': migratePackagePolicyToV7130, '7.14.0': migratePackagePolicyToV7140, + '7.15.0': migratePackagePolicyToV7150, }, }, [PACKAGES_SAVED_OBJECT_TYPE]: { diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts index b4f09e541298a2..e7945077999831 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts @@ -9,3 +9,4 @@ export { migratePackagePolicyToV7110 } from './to_v7_11_0'; export { migratePackagePolicyToV7120 } from './to_v7_12_0'; export { migrateEndpointPackagePolicyToV7130 } from './to_v7_13_0'; export { migrateEndpointPackagePolicyToV7140 } from './to_v7_14_0'; +export { migratePackagePolicyToV7150 } from './to_v7_15_0'; diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.test.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.test.ts new file mode 100644 index 00000000000000..ac0ba62673d606 --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.test.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from 'kibana/server'; + +import type { PackagePolicy } from '../../../../common'; + +import { migratePackagePolicyToV7150 as migration } from './to_v7_15_0'; + +describe('7.15.0 Endpoint Package Policy migration', () => { + const policyDoc = ({ + windowsMemory = {}, + windowsPopup = {}, + windowsMalware = {}, + windowsRansomware = {}, + }) => { + return { + id: 'mock-saved-object-id', + attributes: { + name: 'Some Policy Name', + package: { + name: 'endpoint', + title: '', + version: '', + }, + id: 'endpoint', + policy_id: '', + enabled: true, + namespace: '', + output_id: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + policy: { + value: { + windows: { + ...windowsMalware, + ...windowsRansomware, + ...windowsMemory, + ...windowsPopup, + }, + }, + }, + }, + }, + ], + }, + type: ' nested', + }; + }; + + it('adds windows memory protection alongside malware and ramsomware', () => { + const initialDoc = policyDoc({ + windowsMalware: { malware: { mode: 'off' } }, + windowsRansomware: { ransomware: { mode: 'off', supported: true } }, + windowsPopup: { + popup: { + malware: { + message: '', + enabled: true, + }, + ransomware: { + message: '', + enabled: true, + }, + }, + }, + }); + + const migratedDoc = policyDoc({ + windowsMalware: { malware: { mode: 'off' } }, + windowsRansomware: { ransomware: { mode: 'off', supported: true } }, + // new memory protection + windowsMemory: { memory_protection: { mode: 'off', supported: true } }, + windowsPopup: { + popup: { + malware: { + message: '', + enabled: true, + }, + ransomware: { + message: '', + enabled: true, + }, + // new memory popup setup + memory_protection: { + message: '', + enabled: false, + }, + }, + }, + }); + + expect(migration(initialDoc, {} as SavedObjectMigrationContext)).toEqual(migratedDoc); + }); + + it('does not modify non-endpoint package policies', () => { + const doc: SavedObjectUnsanitizedDoc = { + id: 'mock-saved-object-id', + attributes: { + name: 'Some Policy Name', + package: { + name: 'notEndpoint', + title: '', + version: '', + }, + id: 'notEndpoint', + policy_id: '', + enabled: true, + namespace: '', + output_id: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: 'notEndpoint', + enabled: true, + streams: [], + config: {}, + }, + ], + }, + type: ' nested', + }; + + expect( + migration(doc, {} as SavedObjectMigrationContext) as SavedObjectUnsanitizedDoc + ).toEqual({ + attributes: { + name: 'Some Policy Name', + package: { + name: 'notEndpoint', + title: '', + version: '', + }, + id: 'notEndpoint', + policy_id: '', + enabled: true, + namespace: '', + output_id: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: 'notEndpoint', + enabled: true, + streams: [], + config: {}, + }, + ], + }, + type: ' nested', + id: 'mock-saved-object-id', + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.ts new file mode 100644 index 00000000000000..e820cb47132a69 --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_15_0.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectMigrationFn, SavedObjectUnsanitizedDoc } from 'kibana/server'; +import { cloneDeep } from 'lodash'; + +import type { PackagePolicy } from '../../../../common'; + +export const migratePackagePolicyToV7150: SavedObjectMigrationFn = ( + packagePolicyDoc +) => { + if (packagePolicyDoc.attributes.package?.name !== 'endpoint') { + return packagePolicyDoc; + } + + const updatedPackagePolicyDoc: SavedObjectUnsanitizedDoc = cloneDeep( + packagePolicyDoc + ); + + const input = updatedPackagePolicyDoc.attributes.inputs[0]; + const memory = { + mode: 'off', + // This value is based on license. + // For the migration, we add 'true', our license watcher will correct it, if needed, when the app starts. + supported: true, + }; + const memoryPopup = { + message: '', + enabled: false, + }; + if (input && input.config) { + const policy = input.config.policy.value; + + policy.windows.memory_protection = memory; + policy.windows.popup.memory_protection = memoryPopup; + } + + return updatedPackagePolicyDoc; +}; diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_15_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_15_0.ts new file mode 100644 index 00000000000000..27758296d8d95e --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_15_0.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectMigrationFn } from 'kibana/server'; + +import type { PackagePolicy } from '../../../common'; + +import { migratePackagePolicyToV7150 as SecSolMigratePackagePolicyToV7150 } from './security_solution'; + +export const migratePackagePolicyToV7150: SavedObjectMigrationFn = ( + packagePolicyDoc, + migrationContext +) => { + let updatedPackagePolicyDoc = packagePolicyDoc; + + // Endpoint specific migrations + if (packagePolicyDoc.attributes.package?.name === 'endpoint') { + updatedPackagePolicyDoc = SecSolMigratePackagePolicyToV7150(packagePolicyDoc, migrationContext); + } + + return updatedPackagePolicyDoc; +}; diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts index 63784b8b8b4403..72aae68000e989 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts @@ -29,6 +29,10 @@ export const policyFactory = (): PolicyConfig => { mode: ProtectionModes.prevent, supported: true, }, + memory_protection: { + mode: ProtectionModes.prevent, + supported: true, + }, popup: { malware: { message: '', @@ -38,6 +42,10 @@ export const policyFactory = (): PolicyConfig => { message: '', enabled: true, }, + memory_protection: { + message: '', + enabled: true, + }, }, logging: { file: 'info', @@ -101,6 +109,10 @@ export const policyFactoryWithoutPaidFeatures = ( mode: ProtectionModes.off, supported: false, }, + memory_protection: { + mode: ProtectionModes.off, + supported: false, + }, popup: { ...policy.windows.popup, malware: { @@ -111,6 +123,10 @@ export const policyFactoryWithoutPaidFeatures = ( message: '', enabled: false, }, + memory_protection: { + message: '', + enabled: false, + }, }, }, mac: { @@ -150,6 +166,10 @@ export const policyFactoryWithSupportedFeatures = ( ...policy.windows.ransomware, supported: true, }, + memory_protection: { + ...policy.windows.memory_protection, + supported: true, + }, }, }; }; @@ -157,4 +177,4 @@ export const policyFactoryWithSupportedFeatures = ( /** * Reflects what string the Endpoint will use when message field is default/empty */ -export const DefaultMalwareMessage = 'Elastic Security {action} {filename}'; +export const DefaultPolicyNotificationMessage = 'Elastic Security {action} {filename}'; diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index 076eb51a5fdc54..fd119ba2e4a98c 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -864,6 +864,7 @@ export interface PolicyConfig { security: boolean; }; malware: ProtectionFields; + memory_protection: ProtectionFields & SupportedFields; ransomware: ProtectionFields & SupportedFields; logging: { file: string; @@ -877,6 +878,10 @@ export interface PolicyConfig { message: string; enabled: boolean; }; + memory_protection: { + message: string; + enabled: boolean; + }; }; antivirus_registration: { enabled: boolean; @@ -929,7 +934,13 @@ export interface UIPolicyConfig { */ windows: Pick< PolicyConfig['windows'], - 'events' | 'malware' | 'ransomware' | 'popup' | 'antivirus_registration' | 'advanced' + | 'events' + | 'malware' + | 'ransomware' + | 'popup' + | 'antivirus_registration' + | 'advanced' + | 'memory_protection' >; /** * Mac-specific policy configuration that is supported via the UI diff --git a/x-pack/plugins/security_solution/common/license/policy_config.test.ts b/x-pack/plugins/security_solution/common/license/policy_config.test.ts index 219538184765a4..71668d97e89442 100644 --- a/x-pack/plugins/security_solution/common/license/policy_config.test.ts +++ b/x-pack/plugins/security_solution/common/license/policy_config.test.ts @@ -10,7 +10,7 @@ import { unsetPolicyFeaturesAccordingToLicenseLevel, } from './policy_config'; import { - DefaultMalwareMessage, + DefaultPolicyNotificationMessage, policyFactory, policyFactoryWithSupportedFeatures, policyFactoryWithoutPaidFeatures, @@ -75,55 +75,102 @@ describe('policy_config and licenses', () => { expect(valid).toBeFalsy(); }); - it('allows ransomware to be turned on for Platinum licenses', () => { + it('allows ransomware and memory to be turned on for Platinum licenses', () => { const policy = policyFactoryWithoutPaidFeatures(); policy.windows.ransomware.mode = ProtectionModes.prevent; policy.windows.ransomware.supported = true; + policy.windows.memory_protection.mode = ProtectionModes.prevent; + policy.windows.memory_protection.supported = true; const valid = isEndpointPolicyValidForLicense(policy, Platinum); expect(valid).toBeTruthy(); }); - it('blocks ransomware to be turned on for Gold and below licenses', () => { - const policy = policyFactoryWithoutPaidFeatures(); - policy.windows.ransomware.mode = ProtectionModes.prevent; - - let valid = isEndpointPolicyValidForLicense(policy, Gold); - expect(valid).toBeFalsy(); - valid = isEndpointPolicyValidForLicense(policy, Basic); - expect(valid).toBeFalsy(); - }); - it('allows ransomware notification to be turned on with a Platinum license', () => { + it('allows ransomware and memory_protection notification to be turned on with a Platinum license', () => { const policy = policyFactoryWithoutPaidFeatures(); policy.windows.popup.ransomware.enabled = true; policy.windows.ransomware.supported = true; + policy.windows.popup.memory_protection.enabled = true; + policy.windows.memory_protection.supported = true; const valid = isEndpointPolicyValidForLicense(policy, Platinum); expect(valid).toBeTruthy(); }); - it('blocks ransomware notification to be turned on for Gold and below licenses', () => { - const policy = policyFactoryWithoutPaidFeatures(); - policy.windows.popup.ransomware.enabled = true; - let valid = isEndpointPolicyValidForLicense(policy, Gold); - expect(valid).toBeFalsy(); - valid = isEndpointPolicyValidForLicense(policy, Basic); - expect(valid).toBeFalsy(); + describe('ransomware protection checks', () => { + it('blocks ransomware to be turned on for Gold and below licenses', () => { + const policy = policyFactoryWithoutPaidFeatures(); + policy.windows.ransomware.mode = ProtectionModes.prevent; + + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + + it('blocks ransomware notification to be turned on for Gold and below licenses', () => { + const policy = policyFactoryWithoutPaidFeatures(); + policy.windows.popup.ransomware.enabled = true; + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + + it('allows ransomware notification message changes with a Platinum license', () => { + const policy = policyFactory(); + policy.windows.popup.ransomware.message = 'BOOM'; + const valid = isEndpointPolicyValidForLicense(policy, Platinum); + expect(valid).toBeTruthy(); + }); + it('blocks ransomware notification message changes for Gold and below licenses', () => { + const policy = policyFactory(); + policy.windows.popup.ransomware.message = 'BOOM'; + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); }); - it('allows ransomware notification message changes with a Platinum license', () => { - const policy = policyFactory(); - policy.windows.popup.ransomware.message = 'BOOM'; - const valid = isEndpointPolicyValidForLicense(policy, Platinum); - expect(valid).toBeTruthy(); - }); - it('blocks ransomware notification message changes for Gold and below licenses', () => { - const policy = policyFactory(); - policy.windows.popup.ransomware.message = 'BOOM'; - let valid = isEndpointPolicyValidForLicense(policy, Gold); - expect(valid).toBeFalsy(); - - valid = isEndpointPolicyValidForLicense(policy, Basic); - expect(valid).toBeFalsy(); + describe('memory protection checks', () => { + it('blocks memory_protection to be turned on for Gold and below licenses', () => { + const policy = policyFactoryWithoutPaidFeatures(); + policy.windows.memory_protection.mode = ProtectionModes.prevent; + + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + + it('blocks memory_protection notification to be turned on for Gold and below licenses', () => { + const policy = policyFactoryWithoutPaidFeatures(); + policy.windows.popup.memory_protection.enabled = true; + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + + it('allows memory_protection notification message changes with a Platinum license', () => { + const policy = policyFactory(); + policy.windows.popup.memory_protection.message = 'BOOM'; + const valid = isEndpointPolicyValidForLicense(policy, Platinum); + expect(valid).toBeTruthy(); + }); + + it('blocks memory_protection notification message changes for Gold and below licenses', () => { + const policy = policyFactory(); + policy.windows.popup.memory_protection.message = 'BOOM'; + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); }); it('allows default policyConfig with Basic', () => { @@ -160,6 +207,19 @@ describe('policy_config and licenses', () => { expect(retPolicy.windows.popup.ransomware.message).toEqual(popupMessage); }); + it('does not change any memory fields with a Platinum license', () => { + const policy = policyFactory(); + const popupMessage = 'WOOP WOOP'; + policy.windows.memory_protection.mode = ProtectionModes.detect; + policy.windows.popup.memory_protection.enabled = false; + policy.windows.popup.memory_protection.message = popupMessage; + + const retPolicy = unsetPolicyFeaturesAccordingToLicenseLevel(policy, Platinum); + expect(retPolicy.windows.memory_protection.mode).toEqual(ProtectionModes.detect); + expect(retPolicy.windows.popup.memory_protection.enabled).toBeFalsy(); + expect(retPolicy.windows.popup.memory_protection.message).toEqual(popupMessage); + }); + it('resets Platinum-paid malware fields for lower license tiers', () => { const defaults = policyFactory(); // reference const policy = policyFactory(); // what we will modify, and should be reset @@ -177,7 +237,9 @@ describe('policy_config and licenses', () => { expect(retPolicy.windows.popup.malware.message).not.toEqual(popupMessage); // need to invert the test, since it could be either value - expect(['', DefaultMalwareMessage]).toContain(retPolicy.windows.popup.malware.message); + expect(['', DefaultPolicyNotificationMessage]).toContain( + retPolicy.windows.popup.malware.message + ); }); it('resets Platinum-paid ransomware fields for lower license tiers', () => { @@ -195,7 +257,31 @@ describe('policy_config and licenses', () => { expect(retPolicy.windows.popup.ransomware.message).not.toEqual(popupMessage); // need to invert the test, since it could be either value - expect(['', DefaultMalwareMessage]).toContain(retPolicy.windows.popup.ransomware.message); + expect(['', DefaultPolicyNotificationMessage]).toContain( + retPolicy.windows.popup.ransomware.message + ); + }); + + it('resets Platinum-paid memory_protection fields for lower license tiers', () => { + const defaults = policyFactoryWithoutPaidFeatures(); // reference + const policy = policyFactory(); // what we will modify, and should be reset + const popupMessage = 'WOOP WOOP'; + policy.windows.popup.memory_protection.message = popupMessage; + + const retPolicy = unsetPolicyFeaturesAccordingToLicenseLevel(policy, Gold); + + expect(retPolicy.windows.memory_protection.mode).toEqual( + defaults.windows.memory_protection.mode + ); + expect(retPolicy.windows.popup.memory_protection.enabled).toEqual( + defaults.windows.popup.memory_protection.enabled + ); + expect(retPolicy.windows.popup.memory_protection.message).not.toEqual(popupMessage); + + // need to invert the test, since it could be either value + expect(['', DefaultPolicyNotificationMessage]).toContain( + retPolicy.windows.popup.memory_protection.message + ); }); it('sets ransomware supported field to false when license is below Platinum', () => { @@ -217,6 +303,30 @@ describe('policy_config and licenses', () => { expect(retPolicy.windows.ransomware.supported).toEqual(defaults.windows.ransomware.supported); }); + + it('sets memory_protection supported field to false when license is below Platinum', () => { + const defaults = policyFactoryWithoutPaidFeatures(); // reference + const policy = policyFactory(); // what we will modify, and should be reset + policy.windows.memory_protection.supported = true; + + const retPolicy = unsetPolicyFeaturesAccordingToLicenseLevel(policy, Gold); + + expect(retPolicy.windows.memory_protection.supported).toEqual( + defaults.windows.memory_protection.supported + ); + }); + + it('sets memory_protection supported field to true when license is at Platinum', () => { + const defaults = policyFactoryWithSupportedFeatures(); // reference + const policy = policyFactory(); // what we will modify, and should be reset + policy.windows.memory_protection.supported = false; + + const retPolicy = unsetPolicyFeaturesAccordingToLicenseLevel(policy, Platinum); + + expect(retPolicy.windows.memory_protection.supported).toEqual( + defaults.windows.memory_protection.supported + ); + }); }); describe('policyFactoryWithoutPaidFeatures for gold and below license', () => { diff --git a/x-pack/plugins/security_solution/common/license/policy_config.ts b/x-pack/plugins/security_solution/common/license/policy_config.ts index 171f2d9d0287df..759820df924ef6 100644 --- a/x-pack/plugins/security_solution/common/license/policy_config.ts +++ b/x-pack/plugins/security_solution/common/license/policy_config.ts @@ -9,7 +9,7 @@ import { ILicense } from '../../../licensing/common/types'; import { isAtLeast } from './license'; import { PolicyConfig } from '../endpoint/types'; import { - DefaultMalwareMessage, + DefaultPolicyNotificationMessage, policyFactoryWithoutPaidFeatures, policyFactoryWithSupportedFeatures, } from '../endpoint/models/policy_config'; @@ -30,6 +30,13 @@ export const isEndpointPolicyValidForLicense = ( return false; } + // only platinum or higher may enable ransomware + if ( + policy.windows.memory_protection.supported !== defaults.windows.memory_protection.supported + ) { + return false; + } + return true; // currently, platinum allows all features } @@ -46,12 +53,18 @@ export const isEndpointPolicyValidForLicense = ( // Only Platinum or higher may change the malware message (which can be blank or what Endpoint defaults) if ( [policy.windows, policy.mac].some( - (p) => p.popup.malware.message !== '' && p.popup.malware.message !== DefaultMalwareMessage + (p) => + p.popup.malware.message !== '' && + p.popup.malware.message !== DefaultPolicyNotificationMessage ) ) { return false; } + // only platinum or higher may enable ransomware + if (policy.windows.ransomware.mode !== defaults.windows.ransomware.mode) { + return false; + } // only platinum or higher may enable ransomware if (policy.windows.ransomware.mode !== defaults.windows.ransomware.mode) { return false; @@ -65,7 +78,7 @@ export const isEndpointPolicyValidForLicense = ( // Only Platinum or higher may change the ransomware message (which can be blank or what Endpoint defaults) if ( policy.windows.popup.ransomware.message !== '' && - policy.windows.popup.ransomware.message !== DefaultMalwareMessage + policy.windows.popup.ransomware.message !== DefaultPolicyNotificationMessage ) { return false; } @@ -74,6 +87,31 @@ export const isEndpointPolicyValidForLicense = ( if (policy.windows.ransomware.supported !== defaults.windows.ransomware.supported) { return false; } + // only platinum or higher may enable memory_protection + if (policy.windows.memory_protection.mode !== defaults.windows.memory_protection.mode) { + return false; + } + + // only platinum or higher may enable memory_protection notification + if ( + policy.windows.popup.memory_protection.enabled !== + defaults.windows.popup.memory_protection.enabled + ) { + return false; + } + + // Only Platinum or higher may change the memory_protection message (which can be blank or what Endpoint defaults) + if ( + policy.windows.popup.memory_protection.message !== '' && + policy.windows.popup.memory_protection.message !== DefaultPolicyNotificationMessage + ) { + return false; + } + + // only platinum or higher may enable memory_protection + if (policy.windows.memory_protection.supported !== defaults.windows.memory_protection.supported) { + return false; + } return true; }; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 94208390b660b5..4b3f02ec6f2d0c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -284,6 +284,7 @@ describe('policy details: ', () => { security: true, }, malware: { mode: 'prevent' }, + memory_protection: { mode: 'off', supported: false }, ransomware: { mode: 'off', supported: false }, popup: { malware: { @@ -294,6 +295,10 @@ describe('policy details: ', () => { enabled: false, message: '', }, + memory_protection: { + enabled: false, + message: '', + }, }, logging: { file: 'info' }, antivirus_registration: { diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts index 793d083400aa2f..628d0ee53655f1 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts @@ -6,7 +6,7 @@ */ import { IHttpFetchError } from 'kibana/public'; -import { DefaultMalwareMessage } from '../../../../../../common/endpoint/models/policy_config'; +import { DefaultPolicyNotificationMessage } from '../../../../../../common/endpoint/models/policy_config'; import { PolicyDetailsState, UpdatePolicyResponse } from '../../types'; import { policyIdFromParams, @@ -39,12 +39,17 @@ export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory UIPolicyConfig = createSel events: windows.events, malware: windows.malware, ransomware: windows.ransomware, + memory_protection: windows.memory_protection, popup: windows.popup, antivirus_registration: windows.antivirus_registration, }, diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/types.ts b/x-pack/plugins/security_solution/public/management/pages/policy/types.ts index 269e70b3c24745..fe549b345ee885 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/types.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/types.ts @@ -114,6 +114,12 @@ export type MalwareProtectionOSes = KeysByValueCriteria< { malware: ProtectionFields } >; +/** Returns an array of the policy OSes that have a memory protection field */ +export type MemoryProtectionOSes = KeysByValueCriteria< + UIPolicyConfig, + { memory_protection: ProtectionFields } +>; + /** Returns an array of the policy OSes that have a ransomware protection field */ export type RansomwareProtectionOSes = KeysByValueCriteria< UIPolicyConfig, @@ -121,7 +127,7 @@ export type RansomwareProtectionOSes = KeysByValueCriteria< >; export type PolicyProtection = - | keyof Pick + | keyof Pick | keyof Pick | keyof Pick; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx index 93cf0f370a7159..40448d473ccf95 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx @@ -297,6 +297,16 @@ describe('Policy Details', () => { expect(tooltip).toHaveLength(1); }); + it('memory protection card and user notification checkbox are shown', () => { + const memory = policyView.find('EuiPanel[data-test-subj="memoryProtectionsForm"]'); + const userNotificationCheckbox = policyView.find( + 'EuiCheckbox[data-test-subj="memory_protectionUserNotificationCheckbox"]' + ); + + expect(memory).toHaveLength(1); + expect(userNotificationCheckbox).toHaveLength(1); + }); + it('ransomware card is shown', () => { const ransomware = policyView.find('EuiPanel[data-test-subj="ransomwareProtectionsForm"]'); expect(ransomware).toHaveLength(1); @@ -321,6 +331,15 @@ describe('Policy Details', () => { expect(tooltip).toHaveLength(0); }); + it('memory protection card, and user notification checkbox are hidden', () => { + const memory = policyView.find('EuiPanel[data-test-subj="memoryProtectionsForm"]'); + expect(memory).toHaveLength(0); + const userNotificationCheckbox = policyView.find( + 'EuiCheckbox[data-test-subj="memoryUserNotificationCheckbox"]' + ); + expect(userNotificationCheckbox).toHaveLength(0); + }); + it('ransomware card is hidden', () => { const ransomware = policyView.find('EuiPanel[data-test-subj="ransomwareProtectionsForm"]'); expect(ransomware).toHaveLength(0); @@ -328,7 +347,7 @@ describe('Policy Details', () => { it('shows the locked card in place of 1 paid feature', () => { const lockedCard = policyView.find('EuiCard[data-test-subj="lockedPolicyCard"]'); - expect(lockedCard).toHaveLength(1); + expect(lockedCard).toHaveLength(2); }); }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details_form.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details_form.tsx index 528f3afc1e64ae..dbbf8f2ab53241 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details_form.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details_form.tsx @@ -8,7 +8,9 @@ import { EuiButtonEmpty, EuiSpacer, EuiText } from '@elastic/eui'; import React, { memo, useCallback, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; import { MalwareProtections } from './policy_forms/protections/malware'; +import { MemoryProtection } from './policy_forms/protections/memory'; import { LinuxEvents, MacEvents, WindowsEvents } from './policy_forms/events'; import { AdvancedPolicyForms } from './policy_advanced'; import { AntivirusRegistrationForm } from './components/antivirus_registration_form'; @@ -16,6 +18,20 @@ import { Ransomware } from './policy_forms/protections/ransomware'; import { LockedPolicyCard } from './policy_forms/locked_card'; import { useLicense } from '../../../../common/hooks/use_license'; +const LOCKED_CARD_RAMSOMWARE_TITLE = i18n.translate( + 'xpack.securitySolution.endpoint.policy.details.ransomware', + { + defaultMessage: 'Ransomware', + } +); + +const LOCKED_CARD_MEMORY_TITLE = i18n.translate( + 'xpack.securitySolution.endpoint.policy.details.memory', + { + defaultMessage: 'Memory', + } +); + export const PolicyDetailsForm = memo(() => { const [showAdvancedPolicy, setShowAdvancedPolicy] = useState(false); const handleAdvancedPolicyClick = useCallback(() => { @@ -37,7 +53,13 @@ export const PolicyDetailsForm = memo(() => { - {isPlatinumPlus ? : } + {isPlatinumPlus ? : } + + {isPlatinumPlus ? ( + + ) : ( + + )} diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/locked_card.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/locked_card.tsx index e9e9195b819d34..150ae5e82ef558 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/locked_card.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/locked_card.tsx @@ -30,7 +30,7 @@ const LockedPolicyDiv = styled.div` } `; -export const LockedPolicyCard = memo(() => { +export const LockedPolicyCard = memo(({ title }: { title: string }) => { return ( { icon={} title={

- - - + {title}

} description={false} @@ -67,8 +62,8 @@ export const LockedPolicyCard = memo(() => {

{ + const OSes: Immutable = [OS.windows]; + const protection = 'memory_protection'; + return ( + } + > + + + + + + + + ), + }} + /> + + + ); +}); + +MemoryProtection.displayName = 'MemoryProtection'; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts index e13601c5a2bf22..04042a4f1d9a5f 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts @@ -8,6 +8,7 @@ const popupVersions: Array<[string, string]> = [ ['malware', '7.11+'], ['ransomware', '7.12+'], + ['memory_protection', '7.15+'], ]; export const popupVersionsMap: ReadonlyMap = new Map(popupVersions); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 13cb175ac908e7..eed481aff3c240 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -20005,7 +20005,6 @@ "xpack.securitySolution.endpoint.policy.details.detectionRulesMessage": "{detectionRulesLink}を表示します。事前構築済みルールは、［検出ルール］ページで「Elastic」というタグが付けられています。", "xpack.securitySolution.endpoint.policy.details.eventCollection": "イベント収集", "xpack.securitySolution.endpoint.policy.details.eventCollectionsEnabled": "{selected} / {total} 件のイベント収集が有効です", - "xpack.securitySolution.endpoint.policy.details.lockedCard": "ランサムウェア対策をオンにするには、ライセンスをプラチナに更新するか、30日間の無料トライアルを開始するか、AWS、GCP、またはAzureで{cloudDeploymentLink}にサインアップしてください。", "xpack.securitySolution.endpoint.policy.details.malware": "マルウェア", "xpack.securitySolution.endpoint.policy.details.platinum": "プラチナ", "xpack.securitySolution.endpoint.policy.details.prevent": "防御", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 6e7ab7a33a0f3f..cc288ebecdc747 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -20305,7 +20305,6 @@ "xpack.securitySolution.endpoint.policy.details.detectionRulesMessage": "请查看{detectionRulesLink}。在“检测规则”页面上，预置规则标记有“Elastic”。", "xpack.securitySolution.endpoint.policy.details.eventCollection": "事件收集", "xpack.securitySolution.endpoint.policy.details.eventCollectionsEnabled": "{selected} / {total} 个事件收集已启用", - "xpack.securitySolution.endpoint.policy.details.lockedCard": "要打开勒索软件防护，必须将您的许可证升级到白金级、开始 30 天免费试用或在 AWS、GCP 或 Azure 中实施{cloudDeploymentLink}。", "xpack.securitySolution.endpoint.policy.details.malware": "恶意软件", "xpack.securitySolution.endpoint.policy.details.platinum": "白金级", "xpack.securitySolution.endpoint.policy.details.prevent": "防御", diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index ae60935013d272..6d06b31eddecdd 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -329,12 +329,17 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, logging: { file: 'info' }, malware: { mode: 'prevent' }, + memory_protection: { mode: 'prevent', supported: true }, ransomware: { mode: 'prevent', supported: true }, popup: { malware: { enabled: true, message: 'Elastic Security {action} {filename}', }, + memory_protection: { + enabled: true, + message: 'Elastic Security {action} {filename}', + }, ransomware: { enabled: true, message: 'Elastic Security {action} {filename}', @@ -533,12 +538,17 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, logging: { file: 'info' }, malware: { mode: 'prevent' }, + memory_protection: { mode: 'prevent', supported: true }, ransomware: { mode: 'prevent', supported: true }, popup: { malware: { enabled: true, message: 'Elastic Security {action} {filename}', }, + memory_protection: { + enabled: true, + message: 'Elastic Security {action} {filename}', + }, ransomware: { enabled: true, message: 'Elastic Security {action} {filename}', @@ -734,12 +744,17 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, logging: { file: 'info' }, malware: { mode: 'prevent' }, + memory_protection: { mode: 'prevent', supported: true }, ransomware: { mode: 'prevent', supported: true }, popup: { malware: { enabled: true, message: 'Elastic Security {action} {filename}', }, + memory_protection: { + enabled: true, + message: 'Elastic Security {action} {filename}', + }, ransomware: { enabled: true, message: 'Elastic Security {action} {filename}', From 7101c42bec39eb4ebf905806979673431cc663ed Mon Sep 17 00:00:00 2001 From: Kevin Logan <56395104+kevinlog@users.noreply.github.com> Date: Mon, 19 Jul 2021 12:46:21 -0400 Subject: [PATCH 02/39] [Security Solution] UX fixes for Policy page and Case Host Isolation comment (#106027) * [Security Solution] UX fixes for Policy page and Case Host Isolation comment --- .../components/user_action_tree/helpers.tsx | 31 +++++++++++++++++-- .../user_action_tree/index.test.tsx | 4 +-- .../pages/policy/view/policy_details.tsx | 1 - 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/cases/public/components/user_action_tree/helpers.tsx b/x-pack/plugins/cases/public/components/user_action_tree/helpers.tsx index 422fb8ee581b25..609183aa5c4efc 100644 --- a/x-pack/plugins/cases/public/components/user_action_tree/helpers.tsx +++ b/x-pack/plugins/cases/public/components/user_action_tree/helpers.tsx @@ -5,9 +5,17 @@ * 2.0. */ -import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiLink, EuiCommentProps } from '@elastic/eui'; -import React from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiLink, + EuiCommentProps, + EuiToken, +} from '@elastic/eui'; +import React, { useContext } from 'react'; import classNames from 'classnames'; +import { ThemeContext } from 'styled-components'; import { CaseFullExternalService, ActionConnector, @@ -37,6 +45,7 @@ interface LabelTitle { action: CaseUserActions; field: string; } + export type RuleDetailsNavigation = CasesNavigation; export type ActionsNavigation = CasesNavigation; @@ -365,6 +374,22 @@ export const getGeneratedAlertsAttachment = ({ ), }); +const ActionIcon = React.memo<{ + actionType: string; +}>(({ actionType }) => { + const theme = useContext(ThemeContext); + return ( + + ); +}); + export const getActionAttachment = ({ comment, userCanCrud, @@ -397,7 +422,7 @@ export const getActionAttachment = ({ ), 'data-test-subj': 'endpoint-action', timestamp: , - timelineIcon: comment.actions.type === 'isolate' ? 'lock' : 'lockOpen', + timelineIcon: , actions: ( { ); await waitFor(() => { expect( - wrapper.find(`[data-test-subj="endpoint-action"]`).first().prop('timelineIcon') + wrapper.find(`[data-test-subj="endpoint-action-icon"]`).first().prop('iconType') ).toBe('lock'); }); }); @@ -448,7 +448,7 @@ describe(`UserActionTree`, () => { ); await waitFor(() => { expect( - wrapper.find(`[data-test-subj="endpoint-action"]`).first().prop('timelineIcon') + wrapper.find(`[data-test-subj="endpoint-action-icon"]`).first().prop('iconType') ).toBe('lockOpen'); }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.tsx index b31ec47fdfc49a..8c587858a1f2aa 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.tsx @@ -48,7 +48,6 @@ import { PolicyDetailsForm } from './policy_details_form'; const maxFormWidth = '770px'; const PolicyDetailsHeader = styled.div` padding: ${(props) => props.theme.eui.paddingSizes.xl} 0; - background-color: #fafbfd; border-bottom: 1px solid #d3dae6; .securitySolutionHeaderPage { max-width: ${maxFormWidth}; From 3b921cea56b5646f88551f7ec9ade4532b2b30e9 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen <43350163+qn895@users.noreply.github.com> Date: Mon, 19 Jul 2021 11:49:03 -0500 Subject: [PATCH 03/39] [ML] Fix Index data visualizer sometimes shows wrong doc count for saved searches (#106007) * [ML] Fix flaky saved search * [ML] Re-enable tests * [ML] Make data viz the only tests for flaky test suite runner * [ML] Make ML only * [ML] Remove describe.only Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../index_data_visualizer.tsx | 14 +++++++------- .../ml/data_visualizer/index_data_visualizer.ts | 3 +-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index 8e0230a9bc6f94..2835588625a6e5 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -65,20 +65,16 @@ export const DataVisualizerUrlStateContextProvider: FC { - if (typeof parsedQueryString?.index === 'string') { - const indexPattern = await indexPatterns.get(parsedQueryString.index); - setCurrentIndexPattern(indexPattern); - } - if (typeof parsedQueryString?.savedSearchId === 'string') { const savedSearchId = parsedQueryString.savedSearchId; try { const savedSearch = await savedObjectsClient.get('search', savedSearchId); const indexPatternId = savedSearch.references.find((ref) => ref.type === 'index-pattern') ?.id; - if (indexPatternId !== undefined) { + if (indexPatternId !== undefined && savedSearch) { try { const indexPattern = await indexPatterns.get(indexPatternId); + setCurrentSavedSearch(savedSearch); setCurrentIndexPattern(indexPattern); } catch (e) { toasts.addError(e, { @@ -88,7 +84,6 @@ export const DataVisualizerUrlStateContextProvider: FC { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); From 887a14907626e7a8433b2f7d0557b30becd25a89 Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Mon, 19 Jul 2021 12:54:44 -0400 Subject: [PATCH 04/39] [Alerting] Allow rule to execute if the value is 0 and that mets the condition (#105626) * Allow rule to execute if the value is 0 and that mets the condition * PR feedback * Fix type issue * PR feedback Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../index_threshold/alert_type.test.ts | 205 +++++++++++++++++- .../alert_types/index_threshold/alert_type.ts | 4 +- 2 files changed, 206 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.test.ts index 6584852babd16a..fb9754edae6f1a 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.test.ts @@ -5,19 +5,28 @@ * 2.0. */ +import uuid from 'uuid'; import type { Writable } from '@kbn/utility-types'; import { loggingSystemMock } from '../../../../../../src/core/server/mocks'; -import { getAlertType } from './alert_type'; +import { AlertServices } from '../../../../alerting/server'; +import { getAlertType, ActionGroupId } from './alert_type'; +import { ActionContext } from './action_context'; import { Params } from './alert_type_params'; +import { AlertServicesMock, alertsMock } from '../../../../alerting/server/mocks'; describe('alertType', () => { const logger = loggingSystemMock.create().get(); const data = { timeSeriesQuery: jest.fn(), }; + const alertServices: AlertServicesMock = alertsMock.createAlertServices(); const alertType = getAlertType(logger, Promise.resolve(data)); + afterEach(() => { + data.timeSeriesQuery.mockReset(); + }); + it('alert type creation structure is the expected value', async () => { expect(alertType.id).toBe('.index-threshold'); expect(alertType.name).toBe('Index threshold'); @@ -135,4 +144,198 @@ describe('alertType', () => { `"[aggType]: invalid aggType: \\"foo\\""` ); }); + + it('should ensure 0 results fires actions if it passes the comparator check', async () => { + data.timeSeriesQuery.mockImplementation((...args) => { + return { + results: [ + { + group: 'all documents', + metrics: [['2021-07-14T14:49:30.978Z', 0]], + }, + ], + }; + }); + const params: Params = { + index: 'index-name', + timeField: 'time-field', + aggType: 'foo', + groupBy: 'all', + timeWindowSize: 5, + timeWindowUnit: 'm', + thresholdComparator: '<', + threshold: [1], + }; + + await alertType.executor({ + alertId: uuid.v4(), + startedAt: new Date(), + previousStartedAt: new Date(), + services: (alertServices as unknown) as AlertServices< + {}, + ActionContext, + typeof ActionGroupId + >, + params, + state: { + latestTimestamp: undefined, + }, + spaceId: uuid.v4(), + name: uuid.v4(), + tags: [], + createdBy: null, + updatedBy: null, + rule: { + name: uuid.v4(), + tags: [], + consumer: '', + producer: '', + ruleTypeId: '', + ruleTypeName: '', + enabled: true, + schedule: { + interval: '1h', + }, + actions: [], + createdBy: null, + updatedBy: null, + createdAt: new Date(), + updatedAt: new Date(), + throttle: null, + notifyWhen: null, + }, + }); + + expect(alertServices.alertInstanceFactory).toHaveBeenCalledWith('all documents'); + }); + + it('should ensure a null result does not fire actions', async () => { + const customAlertServices: AlertServicesMock = alertsMock.createAlertServices(); + data.timeSeriesQuery.mockImplementation((...args) => { + return { + results: [ + { + group: 'all documents', + metrics: [['2021-07-14T14:49:30.978Z', null]], + }, + ], + }; + }); + const params: Params = { + index: 'index-name', + timeField: 'time-field', + aggType: 'foo', + groupBy: 'all', + timeWindowSize: 5, + timeWindowUnit: 'm', + thresholdComparator: '<', + threshold: [1], + }; + + await alertType.executor({ + alertId: uuid.v4(), + startedAt: new Date(), + previousStartedAt: new Date(), + services: (customAlertServices as unknown) as AlertServices< + {}, + ActionContext, + typeof ActionGroupId + >, + params, + state: { + latestTimestamp: undefined, + }, + spaceId: uuid.v4(), + name: uuid.v4(), + tags: [], + createdBy: null, + updatedBy: null, + rule: { + name: uuid.v4(), + tags: [], + consumer: '', + producer: '', + ruleTypeId: '', + ruleTypeName: '', + enabled: true, + schedule: { + interval: '1h', + }, + actions: [], + createdBy: null, + updatedBy: null, + createdAt: new Date(), + updatedAt: new Date(), + throttle: null, + notifyWhen: null, + }, + }); + + expect(customAlertServices.alertInstanceFactory).not.toHaveBeenCalled(); + }); + + it('should ensure an undefined result does not fire actions', async () => { + const customAlertServices: AlertServicesMock = alertsMock.createAlertServices(); + data.timeSeriesQuery.mockImplementation((...args) => { + return { + results: [ + { + group: 'all documents', + metrics: [['2021-07-14T14:49:30.978Z', undefined]], + }, + ], + }; + }); + const params: Params = { + index: 'index-name', + timeField: 'time-field', + aggType: 'foo', + groupBy: 'all', + timeWindowSize: 5, + timeWindowUnit: 'm', + thresholdComparator: '<', + threshold: [1], + }; + + await alertType.executor({ + alertId: uuid.v4(), + startedAt: new Date(), + previousStartedAt: new Date(), + services: (customAlertServices as unknown) as AlertServices< + {}, + ActionContext, + typeof ActionGroupId + >, + params, + state: { + latestTimestamp: undefined, + }, + spaceId: uuid.v4(), + name: uuid.v4(), + tags: [], + createdBy: null, + updatedBy: null, + rule: { + name: uuid.v4(), + tags: [], + consumer: '', + producer: '', + ruleTypeId: '', + ruleTypeName: '', + enabled: true, + schedule: { + interval: '1h', + }, + actions: [], + createdBy: null, + updatedBy: null, + createdAt: new Date(), + updatedAt: new Date(), + throttle: null, + notifyWhen: null, + }, + }); + + expect(customAlertServices.alertInstanceFactory).not.toHaveBeenCalled(); + }); }); diff --git a/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts index aa56951b5dcba0..035d999699d4b8 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/index_threshold/alert_type.ts @@ -18,7 +18,7 @@ import { import { ComparatorFns, getHumanReadableComparator } from '../lib'; export const ID = '.index-threshold'; -const ActionGroupId = 'threshold met'; +export const ActionGroupId = 'threshold met'; export function getAlertType( logger: Logger, @@ -180,7 +180,7 @@ export function getAlertType( groupResult.metrics && groupResult.metrics.length > 0 ? groupResult.metrics[0] : null; const value = metric && metric.length === 2 ? metric[1] : null; - if (!value) { + if (value === null || value === undefined) { logger.debug( `alert ${ID}:${alertId} "${name}": no metrics found for group ${instanceId}} from groupResult ${JSON.stringify( groupResult From 595056060aa467bd9e673f150babb034768c7ddc Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Mon, 19 Jul 2021 19:41:01 +0200 Subject: [PATCH 05/39] Parameterize migration test for kibana version (#105417) --- .../migration_7_13_0_unknown_types.test.ts | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts index a30b3d291e7ec9..33f7fec167f2d8 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts @@ -13,7 +13,12 @@ import * as kbnTestServer from '../../../../test_helpers/kbn_server'; import { Root } from '../../../root'; import JSON5 from 'json5'; import { ElasticsearchClient } from '../../../elasticsearch'; +import { Env } from '@kbn/config'; +import { REPO_ROOT } from '@kbn/utils'; +import { getEnvOptions } from '@kbn/config/target/mocks'; +const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; +const targetIndex = `.kibana_${kibanaVersion}_001`; const logFilePath = Path.join(__dirname, '7_13_unknown_types_test.log'); async function removeLogFile() { @@ -76,7 +81,7 @@ describe('migration v2', () => { unknownDocsWarningLog.message.startsWith( '[.kibana] CHECK_UNKNOWN_DOCUMENTS Upgrades will fail for 8.0+ because documents were found for unknown saved ' + 'object types. To ensure that upgrades will succeed in the future, either re-enable plugins or delete ' + - 'these documents from the ".kibana_8.0.0_001" index after the current upgrade completes.' + `these documents from the "${targetIndex}" index after the current upgrade completes.` ) ).toBeTruthy(); @@ -100,14 +105,15 @@ describe('migration v2', () => { }); const client: ElasticsearchClient = esServer.es.getClient(); - const { body: response } = await client.indices.getSettings({ index: '.kibana_8.0.0_001' }); - const settings = response['.kibana_8.0.0_001'] - .settings as estypes.IndicesIndexStatePrefixedSettings; + const { body: response } = await client.indices.getSettings({ + index: targetIndex, + }); + const settings = response[targetIndex].settings as estypes.IndicesIndexStatePrefixedSettings; expect(settings.index).not.toBeUndefined(); expect(settings.index!.blocks?.write).not.toEqual('true'); // Ensure that documents for unknown types were preserved in target index in an unmigrated state - const spaceDocs = await fetchDocs(client, '.kibana_8.0.0_001', 'space'); + const spaceDocs = await fetchDocs(client, targetIndex, 'space'); expect(spaceDocs.map((s) => s.id)).toEqual( expect.arrayContaining([ 'space:default', @@ -123,7 +129,7 @@ describe('migration v2', () => { expect(d.migrationVersion.space).toEqual('6.6.0'); expect(d.coreMigrationVersion).toEqual('7.13.0'); }); - const fooDocs = await fetchDocs(client, '.kibana_8.0.0_001', 'foo'); + const fooDocs = await fetchDocs(client, targetIndex, 'foo'); expect(fooDocs.map((f) => f.id)).toEqual( expect.arrayContaining([ 'P2SQfHkBs3dBRGh--No5', @@ -155,13 +161,13 @@ describe('migration v2', () => { namespaceType: 'agnostic', migrations: { '6.6.0': (d) => d, - '8.0.0': (d) => d, + [kibanaVersion]: (d) => d, }, }); await root.start(); const client: ElasticsearchClient = esServer.es.getClient(); - const spacesDocsMigrated = await fetchDocs(client, '.kibana_8.0.0_001', 'space'); + const spacesDocsMigrated = await fetchDocs(client, targetIndex, 'space'); expect(spacesDocsMigrated.map((s) => s.id)).toEqual( expect.arrayContaining([ 'space:default', @@ -174,12 +180,12 @@ describe('migration v2', () => { ]) ); spacesDocsMigrated.forEach((d) => { - expect(d.migrationVersion.space).toEqual('8.0.0'); // should be migrated - expect(d.coreMigrationVersion).toEqual('8.0.0'); + expect(d.migrationVersion.space).toEqual(kibanaVersion); // should be migrated + expect(d.coreMigrationVersion).toEqual(kibanaVersion); }); // Make sure unmigrated foo docs are also still there in an unmigrated state - const fooDocsUnmigrated = await fetchDocs(client, '.kibana_8.0.0_001', 'foo'); + const fooDocsUnmigrated = await fetchDocs(client, targetIndex, 'foo'); expect(fooDocsUnmigrated.map((f) => f.id)).toEqual( expect.arrayContaining([ 'P2SQfHkBs3dBRGh--No5', From 44f7a99e6f3d53998227614cf10c33e4a4889bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Mon, 19 Jul 2021 19:56:06 +0200 Subject: [PATCH 06/39] [Observability RAC] Remove indexing of rule evaluation documents (#104970) --- x-pack/plugins/rule_registry/server/index.ts | 2 +- .../create_rule_data_client_mock.ts | 31 +- .../server/rule_data_client/types.ts | 1 + .../utils/create_lifecycle_executor.test.ts | 374 ++++++++++++++++++ .../server/utils/create_lifecycle_executor.ts | 19 +- .../utils/create_lifecycle_rule_type.test.ts | 48 +-- .../server/utils/get_rule_executor_data.ts | 15 - 7 files changed, 400 insertions(+), 90 deletions(-) create mode 100644 x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index 6b0765e71cbada..f8d9dec3ea83a4 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -13,7 +13,7 @@ export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract } export type { RacRequestHandlerContext, RacApiRequestHandlerContext } from './types'; export { RuleDataClient } from './rule_data_client'; export { IRuleDataClient } from './rule_data_client/types'; -export { getRuleExecutorData, RuleExecutorData } from './utils/get_rule_executor_data'; +export { getRuleData, RuleExecutorData } from './utils/get_rule_executor_data'; export { createLifecycleRuleTypeFactory, LifecycleAlertService, diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts b/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts index 59f740e0afb731..24b06439fe573a 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts @@ -4,24 +4,31 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { Assign } from '@kbn/utility-types'; +import { PublicContract } from '@kbn/utility-types'; import type { RuleDataClient } from '.'; import { RuleDataReader, RuleDataWriter } from './types'; type MockInstances> = { [K in keyof T]: T[K] extends (...args: infer TArgs) => infer TReturn - ? jest.MockInstance + ? jest.MockInstance & T[K] : never; }; -export function createRuleDataClientMock() { +type RuleDataClientMock = jest.Mocked< + Omit, 'getWriter' | 'getReader'> +> & { + getWriter: (...args: Parameters) => MockInstances; + getReader: (...args: Parameters) => MockInstances; +}; + +export function createRuleDataClientMock(): RuleDataClientMock { const bulk = jest.fn(); const search = jest.fn(); const getDynamicIndexPattern = jest.fn(); - return ({ - createOrUpdateWriteTarget: jest.fn(({ namespace }) => Promise.resolve()), - getReader: jest.fn(() => ({ + return { + createWriteTargetIfNeeded: jest.fn(({}) => Promise.resolve()), + getReader: jest.fn((_options?: { namespace?: string }) => ({ getDynamicIndexPattern, search, })), @@ -29,15 +36,5 @@ export function createRuleDataClientMock() { bulk, })), isWriteEnabled: jest.fn(() => true), - } as unknown) as Assign< - RuleDataClient & Omit, 'options' | 'getClusterClient'>, - { - getWriter: ( - ...args: Parameters - ) => MockInstances; - getReader: ( - ...args: Parameters - ) => MockInstances; - } - >; + }; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index 54e9a1b3c9a6f3..92ba5c7060ebbf 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -35,6 +35,7 @@ export interface RuleDataWriter { export interface IRuleDataClient { getReader(options?: { namespace?: string }): RuleDataReader; getWriter(options?: { namespace?: string }): RuleDataWriter; + isWriteEnabled(): boolean; createWriteTargetIfNeeded(options: { namespace?: string }): Promise; } diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts new file mode 100644 index 00000000000000..a036f42739998c --- /dev/null +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts @@ -0,0 +1,374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock } from '@kbn/logging/target/mocks'; +import { + elasticsearchServiceMock, + savedObjectsClientMock, +} from '../../../../../src/core/server/mocks'; +import { + AlertExecutorOptions, + AlertInstanceContext, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../../alerting/server'; +import { alertsMock } from '../../../alerting/server/mocks'; +import { + ALERT_ID, + ALERT_STATUS, + EVENT_ACTION, + EVENT_KIND, +} from '../../common/technical_rule_data_field_names'; +import { createRuleDataClientMock } from '../rule_data_client/create_rule_data_client_mock'; +import { createLifecycleExecutor } from './create_lifecycle_executor'; + +describe('createLifecycleExecutor', () => { + it('wraps and unwraps the original executor state', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async (options) => { + expect(options.state).toEqual(initialRuleState); + + const nextRuleState: TestRuleState = { + aRuleStateKey: 'NEXT_RULE_STATE_VALUE', + }; + + return nextRuleState; + }); + + const newRuleState = await executor( + createDefaultAlertExecutorOptions({ + params: {}, + state: { wrapped: initialRuleState, trackedAlerts: {} }, + }) + ); + + expect(newRuleState).toEqual({ + wrapped: { + aRuleStateKey: 'NEXT_RULE_STATE_VALUE', + }, + trackedAlerts: {}, + }); + }); + + it('writes initial documents for newly firing alerts', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async ({ services, state }) => { + services.alertWithLifecycle({ + id: 'TEST_ALERT_0', + fields: {}, + }); + services.alertWithLifecycle({ + id: 'TEST_ALERT_1', + fields: {}, + }); + + return state; + }); + + await executor( + createDefaultAlertExecutorOptions({ + params: {}, + state: { wrapped: initialRuleState, trackedAlerts: {} }, + }) + ); + + expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith( + expect.objectContaining({ + body: [ + // alert documents + { index: { _id: expect.any(String) } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'open', + [EVENT_KIND]: 'signal', + }), + { index: { _id: expect.any(String) } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'open', + [EVENT_KIND]: 'signal', + }), + ], + }) + ); + expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // evaluation documents + { index: {} }, + expect.objectContaining({ + [EVENT_KIND]: 'event', + }), + ]), + }) + ); + }); + + it('overwrites existing documents for repeatedly firing alerts', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + ruleDataClientMock.getReader().search.mockResolvedValue({ + hits: { + hits: [ + { + fields: { + [ALERT_ID]: 'TEST_ALERT_0', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc + }, + }, + { + fields: { + [ALERT_ID]: 'TEST_ALERT_1', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc + }, + }, + ], + }, + } as any); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async ({ services, state }) => { + services.alertWithLifecycle({ + id: 'TEST_ALERT_0', + fields: {}, + }); + services.alertWithLifecycle({ + id: 'TEST_ALERT_1', + fields: {}, + }); + + return state; + }); + + await executor( + createDefaultAlertExecutorOptions({ + alertId: 'TEST_ALERT_0', + params: {}, + state: { + wrapped: initialRuleState, + trackedAlerts: { + TEST_ALERT_0: { + alertId: 'TEST_ALERT_0', + alertUuid: 'TEST_ALERT_0_UUID', + started: '2020-01-01T12:00:00.000Z', + }, + TEST_ALERT_1: { + alertId: 'TEST_ALERT_1', + alertUuid: 'TEST_ALERT_1_UUID', + started: '2020-01-02T12:00:00.000Z', + }, + }, + }, + }) + ); + + expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith( + expect.objectContaining({ + body: [ + // alert document + { index: { _id: 'TEST_ALERT_0_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'active', + [EVENT_KIND]: 'signal', + }), + { index: { _id: 'TEST_ALERT_1_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'active', + [EVENT_KIND]: 'signal', + }), + ], + }) + ); + expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // evaluation documents + { index: {} }, + expect.objectContaining({ + [EVENT_KIND]: 'event', + }), + ]), + }) + ); + }); + + it('updates existing documents for recovered alerts', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + ruleDataClientMock.getReader().search.mockResolvedValue({ + hits: { + hits: [ + { + fields: { + '@timestamp': '', + [ALERT_ID]: 'TEST_ALERT_0', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must show up in the written doc + }, + }, + { + fields: { + '@timestamp': '', + [ALERT_ID]: 'TEST_ALERT_1', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc + }, + }, + ], + }, + } as any); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async ({ services, state }) => { + // TEST_ALERT_0 has recovered + services.alertWithLifecycle({ + id: 'TEST_ALERT_1', + fields: {}, + }); + + return state; + }); + + await executor( + createDefaultAlertExecutorOptions({ + alertId: 'TEST_ALERT_0', + params: {}, + state: { + wrapped: initialRuleState, + trackedAlerts: { + TEST_ALERT_0: { + alertId: 'TEST_ALERT_0', + alertUuid: 'TEST_ALERT_0_UUID', + started: '2020-01-01T12:00:00.000Z', + }, + TEST_ALERT_1: { + alertId: 'TEST_ALERT_1', + alertUuid: 'TEST_ALERT_1_UUID', + started: '2020-01-02T12:00:00.000Z', + }, + }, + }, + }) + ); + + expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // alert document + { index: { _id: 'TEST_ALERT_0_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_STATUS]: 'closed', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, + [EVENT_ACTION]: 'close', + [EVENT_KIND]: 'signal', + }), + { index: { _id: 'TEST_ALERT_1_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'active', + [EVENT_KIND]: 'signal', + }), + ]), + }) + ); + expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // evaluation documents + { index: {} }, + expect.objectContaining({ + [EVENT_KIND]: 'event', + }), + ]), + }) + ); + }); +}); + +type TestRuleState = Record & { + aRuleStateKey: string; +}; + +const initialRuleState: TestRuleState = { + aRuleStateKey: 'INITIAL_RULE_STATE_VALUE', +}; + +const createDefaultAlertExecutorOptions = < + Params extends AlertTypeParams = never, + State extends AlertTypeState = never, + InstanceState extends AlertInstanceState = {}, + InstanceContext extends AlertInstanceContext = {}, + ActionGroupIds extends string = '' +>({ + alertId = 'ALERT_ID', + ruleName = 'RULE_NAME', + params, + state, + createdAt = new Date(), + startedAt = new Date(), + updatedAt = new Date(), +}: { + alertId?: string; + ruleName?: string; + params: Params; + state: State; + createdAt?: Date; + startedAt?: Date; + updatedAt?: Date; +}): AlertExecutorOptions => ({ + alertId, + createdBy: 'CREATED_BY', + startedAt, + name: ruleName, + rule: { + updatedBy: null, + tags: [], + name: ruleName, + createdBy: null, + actions: [], + enabled: true, + consumer: 'CONSUMER', + producer: 'PRODUCER', + schedule: { interval: '1m' }, + throttle: null, + createdAt, + updatedAt, + notifyWhen: null, + ruleTypeId: 'RULE_TYPE_ID', + ruleTypeName: 'RULE_TYPE_NAME', + }, + tags: [], + params, + spaceId: 'SPACE_ID', + services: { + alertInstanceFactory: alertsMock.createAlertServices() + .alertInstanceFactory, + savedObjectsClient: savedObjectsClientMock.create(), + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + }, + state, + updatedBy: null, + previousStartedAt: null, + namespace: undefined, +}); diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 8df343fb16d434..2a18f28710d0fc 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { Logger } from '@kbn/logging'; +import type { Logger } from '@kbn/logging'; +import type { PublicContract } from '@kbn/utility-types'; import { getOrElse } from 'fp-ts/lib/Either'; import * as rt from 'io-ts'; import { Mutable } from 'utility-types'; @@ -98,7 +99,10 @@ export type WrappedLifecycleRuleState = AlertTypeS trackedAlerts: Record; }; -export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleDataClient) => < +export const createLifecycleExecutor = ( + logger: Logger, + ruleDataClient: PublicContract +) => < Params extends AlertTypeParams = never, State extends AlertTypeState = never, InstanceState extends AlertInstanceState = never, @@ -242,7 +246,7 @@ export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleData ...alertData, ...ruleExecutorData, [TIMESTAMP]: timestamp, - [EVENT_KIND]: 'event', + [EVENT_KIND]: 'signal', [OWNER]: rule.consumer, [ALERT_ID]: alertId, }; @@ -311,14 +315,7 @@ export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleData if (ruleDataClient.isWriteEnabled()) { await ruleDataClient.getWriter().bulk({ - body: eventsToIndex - .flatMap((event) => [{ index: {} }, event]) - .concat( - Array.from(alertEvents.values()).flatMap((event) => [ - { index: { _id: event[ALERT_UUID]! } }, - event, - ]) - ), + body: eventsToIndex.flatMap((event) => [{ index: { _id: event[ALERT_UUID]! } }, event]), }); } } diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index e26e5b00435f8b..11bb48a7440a73 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -173,7 +173,7 @@ describe('createLifecycleRuleTypeFactory', () => { const evaluationDocuments = documents.filter((doc) => doc['event.kind'] === 'event'); const alertDocuments = documents.filter((doc) => doc['event.kind'] === 'signal'); - expect(evaluationDocuments.length).toBe(2); + expect(evaluationDocuments.length).toBe(0); expect(alertDocuments.length).toBe(2); expect( @@ -188,50 +188,6 @@ describe('createLifecycleRuleTypeFactory', () => { expect(documents.map((doc) => omit(doc, 'kibana.rac.alert.uuid'))).toMatchInlineSnapshot(` Array [ - Object { - "@timestamp": "2021-06-16T09:01:00.000Z", - "event.action": "open", - "event.kind": "event", - "kibana.rac.alert.duration.us": 0, - "kibana.rac.alert.id": "opbeans-java", - "kibana.rac.alert.owner": "consumer", - "kibana.rac.alert.producer": "producer", - "kibana.rac.alert.start": "2021-06-16T09:01:00.000Z", - "kibana.rac.alert.status": "open", - "kibana.space_ids": Array [ - "spaceId", - ], - "rule.category": "ruleTypeName", - "rule.id": "ruleTypeId", - "rule.name": "name", - "rule.uuid": "alertId", - "service.name": "opbeans-java", - "tags": Array [ - "tags", - ], - }, - Object { - "@timestamp": "2021-06-16T09:01:00.000Z", - "event.action": "open", - "event.kind": "event", - "kibana.rac.alert.duration.us": 0, - "kibana.rac.alert.id": "opbeans-node", - "kibana.rac.alert.owner": "consumer", - "kibana.rac.alert.producer": "producer", - "kibana.rac.alert.start": "2021-06-16T09:01:00.000Z", - "kibana.rac.alert.status": "open", - "kibana.space_ids": Array [ - "spaceId", - ], - "rule.category": "ruleTypeName", - "rule.id": "ruleTypeId", - "rule.name": "name", - "rule.uuid": "alertId", - "service.name": "opbeans-node", - "tags": Array [ - "tags", - ], - }, Object { "@timestamp": "2021-06-16T09:01:00.000Z", "event.action": "open", @@ -324,7 +280,7 @@ describe('createLifecycleRuleTypeFactory', () => { const evaluationDocuments = documents.filter((doc) => doc['event.kind'] === 'event'); const alertDocuments = documents.filter((doc) => doc['event.kind'] === 'signal'); - expect(evaluationDocuments.length).toBe(2); + expect(evaluationDocuments.length).toBe(0); expect(alertDocuments.length).toBe(2); expect( diff --git a/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts b/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts index 7cb02428322a65..144c0dafa3786e 100644 --- a/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts +++ b/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts @@ -14,7 +14,6 @@ import { RULE_UUID, TAGS, } from '../../common/technical_rule_data_field_names'; -import { AlertTypeExecutor, AlertTypeWithExecutor } from '../types'; export interface RuleExecutorData { [RULE_CATEGORY]: string; @@ -25,20 +24,6 @@ export interface RuleExecutorData { [TAGS]: string[]; } -export function getRuleExecutorData( - type: AlertTypeWithExecutor, - options: Parameters[0] -) { - return { - [RULE_ID]: type.id, - [RULE_UUID]: options.alertId, - [RULE_CATEGORY]: type.name, - [RULE_NAME]: options.name, - [TAGS]: options.tags, - [PRODUCER]: type.producer, - }; -} - export function getRuleData(options: AlertExecutorOptions) { return { [RULE_ID]: options.rule.ruleTypeId, From e4a8363087cda014e0e76070d2345133f094d560 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Mon, 19 Jul 2021 11:51:36 -0700 Subject: [PATCH 07/39] Remove beta admonitions for Fleet docs (#106010) --- docs/fleet/fleet.asciidoc | 2 -- docs/settings/fleet-settings.asciidoc | 2 -- docs/setup/connect-to-elasticsearch.asciidoc | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/fleet/fleet.asciidoc b/docs/fleet/fleet.asciidoc index 4777800ce5d578..abfc14b55e0200 100644 --- a/docs/fleet/fleet.asciidoc +++ b/docs/fleet/fleet.asciidoc @@ -3,8 +3,6 @@ [[fleet]] = {fleet} -beta[] - {fleet} in {kib} enables you to add and manage integrations for popular services and platforms, as well as manage {elastic-agent} installations in standalone or {fleet} mode. diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 134d9de3f49d88..cb80165e709904 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -5,8 +5,6 @@ {fleet} settings ++++ -experimental[] - You can configure `xpack.fleet` settings in your `kibana.yml`. By default, {fleet} is enabled. To use {fleet}, you also need to configure {kib} and {es} hosts. diff --git a/docs/setup/connect-to-elasticsearch.asciidoc b/docs/setup/connect-to-elasticsearch.asciidoc index 5e18d934863aa9..880c98902983ff 100644 --- a/docs/setup/connect-to-elasticsearch.asciidoc +++ b/docs/setup/connect-to-elasticsearch.asciidoc @@ -27,7 +27,7 @@ image::images/add-data-tutorials.png[Add Data tutorials] [discrete] === Add Elastic Agent -beta[] *Elastic Agent* is a sneak peek at the next generation of +*Elastic Agent* is the next generation of data integration modules, offering a centralized way to set up your integrations. With *Fleet*, you can add From cb7187f71f68e9cbb860cf2875cb6a899451f5c9 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 19 Jul 2021 12:11:50 -0700 Subject: [PATCH 08/39] [Metrics UI] Drop partial buckets from ALL Metrics UI queries (#104784) * [Metrics UI] Change dropLastBucket to dropPartialBuckets - Change offset calculation to millisecond percission - Change dropLastBucket to dropPartialBuckets - Impliment partial bucket filter - Adding partial bucket filter to metric threshold alerts * Cleaning up getElasticsearchMetricQuery * Change timestamp to from_as_string to align to how date_histgram works * Fixing tests to be more realistic * fixing types; removing extra imports * Fixing new mock data to work with previews * Removing value checks since they don't really provide much value * Removing test for refactored functinality * Change value to match millisecond resolution * Fixing values for new partial bucket scheme * removing unused var * Fixing lookback since drops more than last buckets * Changing results count * fixing more tests * Removing empty describe Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../infra/common/http_api/metrics_api.ts | 2 +- .../metric_threshold/lib/evaluate_alert.ts | 89 ++++++-- .../metric_threshold/lib/metric_query.test.ts | 30 +-- .../metric_threshold/lib/metric_query.ts | 21 +- .../metric_threshold_executor.test.ts | 19 +- .../preview_metric_threshold_alert.test.ts | 9 +- .../alerting/metric_threshold/test_mocks.ts | 204 ++++++++++++++---- .../plugins/infra/server/lib/metrics/index.ts | 10 +- ...stogram_buckets_to_timeseries.test.ts.snap | 50 ++++- .../create_aggregations.test.ts.snap | 2 +- .../calculate_date_histogram_offset.test.ts | 32 ++- .../lib/calculate_date_histogram_offset.ts | 7 +- ...rt_histogram_buckets_to_timeseries.test.ts | 53 +++-- ...convert_histogram_buckets_to_timeseries.ts | 13 +- ...ert_request_to_metrics_api_options.test.ts | 2 +- .../convert_request_to_metrics_api_options.ts | 2 +- ...orm_request_to_metrics_api_request.test.ts | 1 + ...ransform_request_to_metrics_api_request.ts | 1 + .../apis/metrics_ui/metrics_alerting.ts | 27 ++- .../apis/metrics_ui/metrics_explorer.ts | 27 +-- .../apis/metrics_ui/snapshot.ts | 12 +- 21 files changed, 435 insertions(+), 178 deletions(-) diff --git a/x-pack/plugins/infra/common/http_api/metrics_api.ts b/x-pack/plugins/infra/common/http_api/metrics_api.ts index e57efd84d22995..94d26abc701fe5 100644 --- a/x-pack/plugins/infra/common/http_api/metrics_api.ts +++ b/x-pack/plugins/infra/common/http_api/metrics_api.ts @@ -35,7 +35,7 @@ export const MetricsAPIRequestRT = rt.intersection([ afterKey: rt.union([rt.null, afterKeyObjectRT]), limit: rt.union([rt.number, rt.null, rt.undefined]), filters: rt.array(rt.object), - dropLastBucket: rt.boolean, + dropPartialBuckets: rt.boolean, alignDataToEnd: rt.boolean, }), ]); diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_alert.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_alert.ts index 144ee6505c5931..aeeb705f9b25a6 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_alert.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_alert.ts @@ -11,6 +11,8 @@ import { isTooManyBucketsPreviewException, TOO_MANY_BUCKETS_PREVIEW_EXCEPTION, } from '../../../../../common/alerting/metrics'; +import { getIntervalInSeconds } from '../../../../utils/get_interval_in_seconds'; +import { roundTimestamp } from '../../../../utils/round_timestamp'; import { InfraSource } from '../../../../../common/source_configuration/source_configuration'; import { InfraDatabaseSearchResponse } from '../../../adapters/framework/adapter_types'; import { createAfterKeyHandler } from '../../../../utils/create_afterkey_handler'; @@ -26,6 +28,7 @@ interface Aggregation { aggregatedValue: { value: number; values?: Array<{ key: number; value: number }> }; doc_count: number; to_as_string: string; + from_as_string: string; key_as_string: string; }>; }; @@ -92,6 +95,8 @@ export const evaluateAlert = value) - .join(', ')]: getValuesFromAggregations(bucket, aggType), + .join(', ')]: getValuesFromAggregations(bucket, aggType, { + from, + to, + bucketSizeInMillis: intervalAsMS, + }), }), {} ); @@ -153,7 +177,8 @@ const getMetric: ( return { [UNGROUPED_FACTORY_KEY]: getValuesFromAggregations( (result.aggregations! as unknown) as Aggregation, - aggType + aggType, + { from, to, bucketSizeInMillis: intervalAsMS } ), }; } catch (e) { @@ -173,31 +198,55 @@ const getMetric: ( } }; +interface DropPartialBucketOptions { + from: number; + to: number; + bucketSizeInMillis: number; +} + +const dropPartialBuckets = ({ from, to, bucketSizeInMillis }: DropPartialBucketOptions) => ( + row: { + key: string; + value: number; + } | null +) => { + if (row == null) return null; + const timestamp = new Date(row.key).valueOf(); + return timestamp >= from && timestamp + bucketSizeInMillis <= to; +}; + const getValuesFromAggregations = ( aggregations: Aggregation, - aggType: MetricExpressionParams['aggType'] + aggType: MetricExpressionParams['aggType'], + dropPartialBucketsOptions: DropPartialBucketOptions ) => { try { const { buckets } = aggregations.aggregatedIntervals; if (!buckets.length) return null; // No Data state if (aggType === Aggregators.COUNT) { - return buckets.map((bucket) => ({ - key: bucket.to_as_string, - value: bucket.doc_count, - })); + return buckets + .map((bucket) => ({ + key: bucket.from_as_string, + value: bucket.doc_count, + })) + .filter(dropPartialBuckets(dropPartialBucketsOptions)); } if (aggType === Aggregators.P95 || aggType === Aggregators.P99) { - return buckets.map((bucket) => { - const values = bucket.aggregatedValue?.values || []; - const firstValue = first(values); - if (!firstValue) return null; - return { key: bucket.to_as_string, value: firstValue.value }; - }); + return buckets + .map((bucket) => { + const values = bucket.aggregatedValue?.values || []; + const firstValue = first(values); + if (!firstValue) return null; + return { key: bucket.from_as_string, value: firstValue.value }; + }) + .filter(dropPartialBuckets(dropPartialBucketsOptions)); } - return buckets.map((bucket) => ({ - key: bucket.key_as_string ?? bucket.to_as_string, - value: bucket.aggregatedValue?.value ?? null, - })); + return buckets + .map((bucket) => ({ + key: bucket.key_as_string ?? bucket.from_as_string, + value: bucket.aggregatedValue?.value ?? null, + })) + .filter(dropPartialBuckets(dropPartialBucketsOptions)); } catch (e) { return NaN; // Error state } diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts index 79aa94f98d2ada..2ba8365d6b4a91 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts @@ -7,6 +7,7 @@ import { MetricExpressionParams } from '../types'; import { getElasticsearchMetricQuery } from './metric_query'; +import moment from 'moment'; describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { const expressionParams = { @@ -18,9 +19,13 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { const timefield = '@timestamp'; const groupBy = 'host.doggoname'; + const timeframe = { + start: moment().subtract(5, 'minutes').valueOf(), + end: moment().valueOf(), + }; describe('when passed no filterQuery', () => { - const searchBody = getElasticsearchMetricQuery(expressionParams, timefield, groupBy); + const searchBody = getElasticsearchMetricQuery(expressionParams, timefield, timeframe, groupBy); test('includes a range filter', () => { expect( searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range')) @@ -43,6 +48,7 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { const searchBody = getElasticsearchMetricQuery( expressionParams, timefield, + timeframe, groupBy, filterQuery ); @@ -58,26 +64,4 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { ); }); }); - - describe('handles time', () => { - const end = new Date('2020-07-08T22:07:27.235Z').valueOf(); - const timerange = { - end, - start: end - 5 * 60 * 1000, - }; - const searchBody = getElasticsearchMetricQuery( - expressionParams, - timefield, - undefined, - undefined, - timerange - ); - test('by rounding timestamps to the nearest timeUnit', () => { - const rangeFilter = searchBody.query.bool.filter.find((filter) => - filter.hasOwnProperty('range') - )?.range[timefield]; - expect(rangeFilter?.lte).toBe(1594246020000); - expect(rangeFilter?.gte).toBe(1594245720000); - }); - }); }); diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts index 0e495c08cc9fd7..a9fefd57d01e6e 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts @@ -8,11 +8,9 @@ import { networkTraffic } from '../../../../../common/inventory_models/shared/metrics/snapshot/network_traffic'; import { MetricExpressionParams, Aggregators } from '../types'; import { getIntervalInSeconds } from '../../../../utils/get_interval_in_seconds'; -import { roundTimestamp } from '../../../../utils/round_timestamp'; import { createPercentileAggregation } from './create_percentile_aggregation'; import { calculateDateHistogramOffset } from '../../../metrics/lib/calculate_date_histogram_offset'; -const MINIMUM_BUCKETS = 5; const COMPOSITE_RESULTS_PER_PAGE = 100; const getParsedFilterQuery: (filterQuery: string | undefined) => Record | null = ( @@ -25,9 +23,9 @@ const getParsedFilterQuery: (filterQuery: string | undefined) => Record { if (aggType === Aggregators.COUNT && metric) { throw new Error('Cannot aggregate document count with a metric'); @@ -38,19 +36,10 @@ export const getElasticsearchMetricQuery = ( const interval = `${timeSize}${timeUnit}`; const intervalAsSeconds = getIntervalInSeconds(interval); const intervalAsMS = intervalAsSeconds * 1000; - - const to = roundTimestamp(timeframe ? timeframe.end : Date.now(), timeUnit); - // We need enough data for 5 buckets worth of data. We also need - // to convert the intervalAsSeconds to milliseconds. - const minimumFrom = to - intervalAsMS * MINIMUM_BUCKETS; - - const from = roundTimestamp( - timeframe && timeframe.start <= minimumFrom ? timeframe.start : minimumFrom, - timeUnit - ); - + const to = timeframe.end; + const from = timeframe.start; const offset = calculateDateHistogramOffset({ from, to, interval, field: timefield }); - const offsetInMS = parseInt(offset, 10) * 1000; + const offsetInMS = parseInt(offset, 10); const aggregations = aggType === Aggregators.COUNT diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts index e5cad99dcb4ed4..0dda299704d528 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts @@ -119,14 +119,12 @@ describe('The metric threshold alert type', () => { expect(mostRecentAction(instanceID)).toBe(undefined); }); test('reports expected values to the action context', async () => { - const now = 1577858400000; await execute(Comparator.GT, [0.75]); const { action } = mostRecentAction(instanceID); expect(action.group).toBe('*'); expect(action.reason).toContain('current value is 1'); expect(action.reason).toContain('threshold of 0.75'); expect(action.reason).toContain('test.metric.1'); - expect(action.timestamp).toBe(new Date(now).toISOString()); }); }); @@ -428,7 +426,6 @@ describe('The metric threshold alert type', () => { }, }); test('reports values converted from decimals to percentages to the action context', async () => { - const now = 1577858400000; await execute(); const { action } = mostRecentAction(instanceID); expect(action.group).toBe('*'); @@ -436,7 +433,6 @@ describe('The metric threshold alert type', () => { expect(action.reason).toContain('threshold of 75%'); expect(action.threshold.condition0[0]).toBe('75%'); expect(action.value.condition0).toBe('100%'); - expect(action.timestamp).toBe(new Date(now).toISOString()); }); }); }); @@ -460,7 +456,8 @@ const executor = createMetricThresholdExecutor(mockLibs); const services: AlertServicesMock = alertsMock.createAlertServices(); services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: any): any => { - if (params.index === 'alternatebeat-*') return mocks.changedSourceIdResponse; + const from = params?.body.query.bool.filter[0]?.range['@timestamp'].gte; + if (params.index === 'alternatebeat-*') return mocks.changedSourceIdResponse(from); const metric = params?.body.query.bool.filter[1]?.exists.field; if (params?.body.aggs.groupings) { if (params?.body.aggs.groupings.composite.after) { @@ -470,25 +467,27 @@ services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: a } if (metric === 'test.metric.2') { return elasticsearchClientMock.createSuccessTransportRequestPromise( - mocks.alternateCompositeResponse + mocks.alternateCompositeResponse(from) ); } return elasticsearchClientMock.createSuccessTransportRequestPromise( - mocks.basicCompositeResponse + mocks.basicCompositeResponse(from) ); } if (metric === 'test.metric.2') { return elasticsearchClientMock.createSuccessTransportRequestPromise( - mocks.alternateMetricResponse + mocks.alternateMetricResponse(from) ); } else if (metric === 'test.metric.3') { return elasticsearchClientMock.createSuccessTransportRequestPromise( - params?.body.aggs.aggregatedIntervals.aggregations.aggregatedValue_max + params?.body.aggs.aggregatedIntervals.aggregations.aggregatedValueMax ? mocks.emptyRateResponse : mocks.emptyMetricResponse ); } - return elasticsearchClientMock.createSuccessTransportRequestPromise(mocks.basicMetricResponse); + return elasticsearchClientMock.createSuccessTransportRequestPromise( + mocks.basicMetricResponse(from) + ); }); services.savedObjectsClient.get.mockImplementation(async (type: string, sourceId: string) => { if (sourceId === 'alternate') diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/preview_metric_threshold_alert.test.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/preview_metric_threshold_alert.test.ts index 49cb8d70f6020a..e03b475191dff1 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/preview_metric_threshold_alert.test.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/preview_metric_threshold_alert.test.ts @@ -167,6 +167,7 @@ describe('Previewing the metric threshold alert type', () => { const services: AlertServicesMock = alertsMock.createAlertServices(); services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: any): any => { + const from = params?.body.query.bool.filter[0]?.range['@timestamp'].gte; const metric = params?.body.query.bool.filter[1]?.exists.field; if (params?.body.aggs.groupings) { if (params?.body.aggs.groupings.composite.after) { @@ -175,21 +176,21 @@ services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: a ); } return elasticsearchClientMock.createSuccessTransportRequestPromise( - mocks.basicCompositePreviewResponse + mocks.basicCompositePreviewResponse(from) ); } if (metric === 'test.metric.2') { return elasticsearchClientMock.createSuccessTransportRequestPromise( - mocks.alternateMetricPreviewResponse + mocks.alternateMetricPreviewResponse(from) ); } if (metric === 'test.metric.3') { return elasticsearchClientMock.createSuccessTransportRequestPromise( - mocks.repeatingMetricPreviewResponse + mocks.repeatingMetricPreviewResponse(from) ); } return elasticsearchClientMock.createSuccessTransportRequestPromise( - mocks.basicMetricPreviewResponse + mocks.basicMetricPreviewResponse(from) ); }); diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts index 47da539afea191..5af14b3fbd17d6 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts @@ -4,64 +4,176 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -const bucketsA = [ +import { range } from 'lodash'; +const bucketsA = (from: number) => [ + { + doc_count: null, + aggregatedValue: { value: null, values: [{ key: 95.0, value: null }] }, + from_as_string: new Date(from).toISOString(), + }, + { + doc_count: 2, + aggregatedValue: { value: 0.5, values: [{ key: 95.0, value: 0.5 }] }, + from_as_string: new Date(from + 60000).toISOString(), + }, { doc_count: 2, aggregatedValue: { value: 0.5, values: [{ key: 95.0, value: 0.5 }] }, + from_as_string: new Date(from + 120000).toISOString(), + }, + { + doc_count: 2, + aggregatedValue: { value: 0.5, values: [{ key: 95.0, value: 0.5 }] }, + from_as_string: new Date(from + 180000).toISOString(), }, { doc_count: 3, aggregatedValue: { value: 1.0, values: [{ key: 95.0, value: 1.0 }] }, - to_as_string: new Date(1577858400000).toISOString(), + from_as_string: new Date(from + 240000).toISOString(), + }, + { + doc_count: 1, + aggregatedValue: { value: 1.0, values: [{ key: 95.0, value: 1.0 }] }, + from_as_string: new Date(from + 300000).toISOString(), }, ]; -const bucketsB = [ +const bucketsB = (from: number) => [ + { + doc_count: 0, + aggregatedValue: { value: null, values: [{ key: 99.0, value: null }] }, + from_as_string: new Date(from).toISOString(), + }, + { + doc_count: 4, + aggregatedValue: { value: 2.5, values: [{ key: 99.0, value: 2.5 }] }, + from_as_string: new Date(from + 60000).toISOString(), + }, { doc_count: 4, aggregatedValue: { value: 2.5, values: [{ key: 99.0, value: 2.5 }] }, + from_as_string: new Date(from + 120000).toISOString(), + }, + { + doc_count: 4, + aggregatedValue: { value: 2.5, values: [{ key: 99.0, value: 2.5 }] }, + from_as_string: new Date(from + 180000).toISOString(), }, { doc_count: 5, aggregatedValue: { value: 3.5, values: [{ key: 99.0, value: 3.5 }] }, + from_as_string: new Date(from + 240000).toISOString(), + }, + { + doc_count: 1, + aggregatedValue: { value: 3, values: [{ key: 99.0, value: 3 }] }, + from_as_string: new Date(from + 300000).toISOString(), }, ]; -const bucketsC = [ +const bucketsC = (from: number) => [ + { + doc_count: 0, + aggregatedValue: { value: null }, + from_as_string: new Date(from).toISOString(), + }, { doc_count: 2, aggregatedValue: { value: 0.5 }, + from_as_string: new Date(from + 60000).toISOString(), + }, + { + doc_count: 2, + aggregatedValue: { value: 0.5 }, + from_as_string: new Date(from + 120000).toISOString(), + }, + { + doc_count: 2, + aggregatedValue: { value: 0.5 }, + from_as_string: new Date(from + 180000).toISOString(), }, { doc_count: 3, - aggregatedValue: { value: 16.0 }, + aggregatedValue: { value: 16 }, + from_as_string: new Date(from + 240000).toISOString(), + }, + { + doc_count: 1, + aggregatedValue: { value: 3 }, + from_as_string: new Date(from + 300000).toISOString(), }, ]; -const previewBucketsA = Array.from(Array(60), (_, i) => bucketsA[i % 2]); // Repeat bucketsA to a total length of 60 -const previewBucketsB = Array.from(Array(60), (_, i) => bucketsB[i % 2]); -const previewBucketsWithNulls = [ - ...Array.from(Array(10), (_, i) => ({ aggregatedValue: { value: null } })), - ...previewBucketsA.slice(10), +const previewBucketsA = (from: number) => + range(from, from + 3600000, 60000).map((timestamp, i) => { + return { + doc_count: i % 2 ? 3 : 2, + aggregatedValue: { value: i % 2 ? 16 : 0.5 }, + from_as_string: new Date(timestamp).toISOString(), + }; + }); + +const previewBucketsB = (from: number) => + range(from, from + 3600000, 60000).map((timestamp, i) => { + const value = i % 2 ? 3.5 : 2.5; + return { + doc_count: i % 2 ? 3 : 2, + aggregatedValue: { value, values: [{ key: 99.0, value }] }, + from_as_string: new Date(timestamp).toISOString(), + }; + }); + +const previewBucketsWithNulls = (from: number) => [ + // 25 Fired + ...range(from, from + 1500000, 60000).map((timestamp) => { + return { + doc_count: 2, + aggregatedValue: { value: 1, values: [{ key: 95.0, value: 1 }] }, + from_as_string: new Date(timestamp).toISOString(), + }; + }), + // 25 OK + ...range(from + 2100000, from + 2940000, 60000).map((timestamp) => { + return { + doc_count: 2, + aggregatedValue: { value: 0.5, values: [{ key: 95.0, value: 0.5 }] }, + from_as_string: new Date(timestamp).toISOString(), + }; + }), + // 10 No Data + ...range(from + 3000000, from + 3600000, 60000).map((timestamp) => { + return { + doc_count: 0, + aggregatedValue: { value: null, values: [{ key: 95.0, value: null }] }, + from_as_string: new Date(timestamp).toISOString(), + }; + }), ]; -const previewBucketsRepeat = Array.from(Array(60), (_, i) => bucketsA[Math.max(0, (i % 3) - 1)]); -export const basicMetricResponse = { +const previewBucketsRepeat = (from: number) => + range(from, from + 3600000, 60000).map((timestamp, i) => { + return { + doc_count: i % 3 ? 3 : 2, + aggregatedValue: { value: i % 3 ? 0.5 : 16 }, + from_as_string: new Date(timestamp).toISOString(), + }; + }); + +export const basicMetricResponse = (from: number) => ({ aggregations: { aggregatedIntervals: { - buckets: bucketsA, + buckets: bucketsA(from), }, }, -}; +}); -export const alternateMetricResponse = { +export const alternateMetricResponse = (from: number) => ({ aggregations: { aggregatedIntervals: { - buckets: bucketsB, + buckets: bucketsB(from), }, }, -}; +}); export const emptyMetricResponse = { aggregations: { @@ -71,21 +183,21 @@ export const emptyMetricResponse = { }, }; -export const emptyRateResponse = { +export const emptyRateResponse = (from: number) => ({ aggregations: { aggregatedIntervals: { buckets: [ { doc_count: 2, - // eslint-disable-next-line @typescript-eslint/naming-convention - aggregatedValue_max: { value: null }, + aggregatedValueMax: { value: null }, + from_as_string: new Date(from).toISOString(), }, ], }, }, -}; +}); -export const basicCompositeResponse = { +export const basicCompositeResponse = (from: number) => ({ aggregations: { groupings: { after_key: { groupBy0: 'foo' }, @@ -95,7 +207,7 @@ export const basicCompositeResponse = { groupBy0: 'a', }, aggregatedIntervals: { - buckets: bucketsA, + buckets: bucketsA(from), }, }, { @@ -103,7 +215,7 @@ export const basicCompositeResponse = { groupBy0: 'b', }, aggregatedIntervals: { - buckets: bucketsB, + buckets: bucketsB(from), }, }, ], @@ -114,9 +226,9 @@ export const basicCompositeResponse = { value: 2, }, }, -}; +}); -export const alternateCompositeResponse = { +export const alternateCompositeResponse = (from: number) => ({ aggregations: { groupings: { after_key: { groupBy0: 'foo' }, @@ -126,7 +238,7 @@ export const alternateCompositeResponse = { groupBy0: 'a', }, aggregatedIntervals: { - buckets: bucketsB, + buckets: bucketsB(from), }, }, { @@ -134,7 +246,7 @@ export const alternateCompositeResponse = { groupBy0: 'b', }, aggregatedIntervals: { - buckets: bucketsA, + buckets: bucketsA(from), }, }, ], @@ -145,46 +257,46 @@ export const alternateCompositeResponse = { value: 2, }, }, -}; +}); export const compositeEndResponse = { aggregations: {}, hits: { total: { value: 0 } }, }; -export const changedSourceIdResponse = { +export const changedSourceIdResponse = (from: number) => ({ aggregations: { aggregatedIntervals: { - buckets: bucketsC, + buckets: bucketsC(from), }, }, -}; +}); -export const basicMetricPreviewResponse = { +export const basicMetricPreviewResponse = (from: number) => ({ aggregations: { aggregatedIntervals: { - buckets: previewBucketsA, + buckets: previewBucketsA(from), }, }, -}; +}); -export const alternateMetricPreviewResponse = { +export const alternateMetricPreviewResponse = (from: number) => ({ aggregations: { aggregatedIntervals: { - buckets: previewBucketsWithNulls, + buckets: previewBucketsWithNulls(from), }, }, -}; +}); -export const repeatingMetricPreviewResponse = { +export const repeatingMetricPreviewResponse = (from: number) => ({ aggregations: { aggregatedIntervals: { - buckets: previewBucketsRepeat, + buckets: previewBucketsRepeat(from), }, }, -}; +}); -export const basicCompositePreviewResponse = { +export const basicCompositePreviewResponse = (from: number) => ({ aggregations: { groupings: { after_key: { groupBy0: 'foo' }, @@ -194,7 +306,7 @@ export const basicCompositePreviewResponse = { groupBy0: 'a', }, aggregatedIntervals: { - buckets: previewBucketsA, + buckets: previewBucketsA(from), }, }, { @@ -202,7 +314,7 @@ export const basicCompositePreviewResponse = { groupBy0: 'b', }, aggregatedIntervals: { - buckets: previewBucketsB, + buckets: previewBucketsB(from), }, }, ], @@ -213,4 +325,4 @@ export const basicCompositePreviewResponse = { value: 2, }, }, -}; +}); diff --git a/x-pack/plugins/infra/server/lib/metrics/index.ts b/x-pack/plugins/infra/server/lib/metrics/index.ts index e436ad2ba0b05f..a2d9ad2b401d47 100644 --- a/x-pack/plugins/infra/server/lib/metrics/index.ts +++ b/x-pack/plugins/infra/server/lib/metrics/index.ts @@ -91,7 +91,12 @@ export const query = async ( return { series: groupings.buckets.map((bucket) => { const keys = Object.values(bucket.key); - return convertHistogramBucketsToTimeseries(keys, options, bucket.histogram.buckets); + return convertHistogramBucketsToTimeseries( + keys, + options, + bucket.histogram.buckets, + bucketSize * 1000 + ); }), info: { afterKey: returnAfterKey ? afterKey : null, @@ -108,7 +113,8 @@ export const query = async ( convertHistogramBucketsToTimeseries( ['*'], options, - response.aggregations.histogram.buckets + response.aggregations.histogram.buckets, + bucketSize * 1000 ), ], info: { diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/convert_histogram_buckets_to_timeseries.test.ts.snap b/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/convert_histogram_buckets_to_timeseries.test.ts.snap index 6d3c0bf71cae78..5ec5166c8deb44 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/convert_histogram_buckets_to_timeseries.test.ts.snap +++ b/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/convert_histogram_buckets_to_timeseries.test.ts.snap @@ -29,6 +29,14 @@ Object { "metric_0": 1, "timestamp": 1577836920000, }, + Object { + "metric_0": 1, + "timestamp": 1577836980000, + }, + Object { + "metric_0": 1, + "timestamp": 1577837040000, + }, ], } `; @@ -62,9 +70,17 @@ Object { "metric_0": 1, "timestamp": 1577836920000, }, + Object { + "metric_0": 1, + "timestamp": 1577836980000, + }, + Object { + "metric_0": 1, + "timestamp": 1577837040000, + }, Object { "metric_0": null, - "timestamp": 1577836920000, + "timestamp": 1577837100000, }, ], } @@ -81,7 +97,7 @@ Object { } `; -exports[`convertHistogramBucketsToTimeseies(keys, options, buckets) should tranform top_metric aggregations 1`] = ` +exports[`convertHistogramBucketsToTimeseies(keys, options, buckets) should transform top_metric aggregations 1`] = ` Object { "columns": Array [ Object { @@ -152,7 +168,15 @@ Object { }, Object { "metric_0": 4, - "timestamp": 1577836920000, + "timestamp": 1577836980000, + }, + Object { + "metric_0": 4, + "timestamp": 1577837040000, + }, + Object { + "metric_0": 4, + "timestamp": 1577837100000, }, ], } @@ -187,9 +211,17 @@ Object { "metric_0": 2, "timestamp": 1577836920000, }, + Object { + "metric_0": 2, + "timestamp": 1577836980000, + }, + Object { + "metric_0": 2, + "timestamp": 1577837040000, + }, Object { "metric_0": null, - "timestamp": 1577836920000, + "timestamp": 1577837100000, }, ], } @@ -226,7 +258,15 @@ Object { }, Object { "metric_0": 3, - "timestamp": 1577836920000, + "timestamp": 1577836980000, + }, + Object { + "metric_0": 3, + "timestamp": 1577837040000, + }, + Object { + "metric_0": 3, + "timestamp": 1577837100000, }, ], } diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap b/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap index 2cbbc623aed38f..c7acfb147aa7e1 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap +++ b/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap @@ -17,7 +17,7 @@ Object { }, "field": "@timestamp", "fixed_interval": "1m", - "offset": "-60s", + "offset": "-60000ms", }, }, } diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.test.ts b/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.test.ts index 7cb76ee088c1a4..ef0f64855acd86 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.test.ts +++ b/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.test.ts @@ -17,6 +17,36 @@ describe('calculateDateHistogramOffset(timerange)', () => { field: '@timestamp', }; const offset = calculateDateHistogramOffset(timerange); - expect(offset).toBe('-28s'); + expect(offset).toBe('-28000ms'); + }); + it('should work with un-even timeranges (60s buckets)', () => { + const timerange = { + from: 1625057349373, + to: 1625057649373, + interval: '60s', + field: '@timestamp', + }; + const offset = calculateDateHistogramOffset(timerange); + expect(offset).toBe('-51373ms'); + }); + it('should work with un-even timeranges (5m buckets)', () => { + const timerange = { + from: 1625516185059, + to: 1625602885059, + interval: '5m', + field: '@timestamp', + }; + const offset = calculateDateHistogramOffset(timerange); + expect(offset).toBe('-215059ms'); + }); + it('should work with un-even timeranges (>=10s buckets)', () => { + const timerange = { + from: 1625516185059, + to: 1625602885059, + interval: '>=10s', + field: '@timestamp', + }; + const offset = calculateDateHistogramOffset(timerange); + expect(offset).toBe('-215059ms'); }); }); diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.ts b/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.ts index 1124b95e606de5..6f35a624a11dac 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.ts +++ b/x-pack/plugins/infra/server/lib/metrics/lib/calculate_date_histogram_offset.ts @@ -14,5 +14,10 @@ export const calculateDateHistogramOffset = (timerange: MetricsAPITimerange): st // negative offset to align buckets with full intervals (e.g. minutes) const offset = (fromInSeconds % bucketSize) - bucketSize; - return `${offset}s`; + + // Because everything is being rounded to the nearest second, except the timerange, + // we need to adjust the buckets to account for the millisecond offset otherwise + // the last bucket will be only contain the difference. + const millisOffset = timerange.to % 1000; + return `${offset * 1000 - millisOffset}ms`; }; diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.test.ts b/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.test.ts index 31478c5f1aa4ce..b49560f8c25f6e 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.test.ts +++ b/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.test.ts @@ -15,7 +15,7 @@ const options: MetricsAPIRequest = { timerange: { field: '@timestamp', from: moment('2020-01-01T00:00:00Z').valueOf(), - to: moment('2020-01-01T01:00:00Z').valueOf(), + to: moment('2020-01-01T00:00:00Z').add(5, 'minute').valueOf(), interval: '1m', }, limit: 9, @@ -45,8 +45,20 @@ const buckets = [ metric_0: { value: 1 }, }, { - key: moment('2020-01-01T00:00:00Z').add(2, 'minute').valueOf(), - key_as_string: moment('2020-01-01T00:00:00Z').add(2, 'minute').toISOString(), + key: moment('2020-01-01T00:00:00Z').add(3, 'minute').valueOf(), + key_as_string: moment('2020-01-01T00:00:00Z').add(3, 'minute').toISOString(), + doc_count: 1, + metric_0: { value: 1 }, + }, + { + key: moment('2020-01-01T00:00:00Z').add(4, 'minute').valueOf(), + key_as_string: moment('2020-01-01T00:00:00Z').add(4, 'minute').toISOString(), + doc_count: 1, + metric_0: { value: 1 }, + }, + { + key: moment('2020-01-01T00:00:00Z').add(5, 'minute').valueOf(), + key_as_string: moment('2020-01-01T00:00:00Z').add(5, 'minute').toISOString(), doc_count: 1, metric_0: { value: null }, }, @@ -54,16 +66,21 @@ const buckets = [ describe('convertHistogramBucketsToTimeseies(keys, options, buckets)', () => { it('should just work', () => { - expect(convertHistogramBucketsToTimeseries(keys, options, buckets)).toMatchSnapshot(); + expect(convertHistogramBucketsToTimeseries(keys, options, buckets, 60000)).toMatchSnapshot(); }); it('should drop the last bucket', () => { expect( - convertHistogramBucketsToTimeseries(keys, { ...options, dropLastBucket: true }, buckets) + convertHistogramBucketsToTimeseries( + keys, + { ...options, dropPartialBuckets: true }, + buckets, + 60000 + ) ).toMatchSnapshot(); }); it('should return empty timeseries for empty metrics', () => { expect( - convertHistogramBucketsToTimeseries(keys, { ...options, metrics: [] }, buckets) + convertHistogramBucketsToTimeseries(keys, { ...options, metrics: [] }, buckets, 60000) ).toMatchSnapshot(); }); it('should work with normalized_values', () => { @@ -75,7 +92,7 @@ describe('convertHistogramBucketsToTimeseies(keys, options, buckets)', () => { return bucket; }); expect( - convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithNormalizedValue) + convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithNormalizedValue, 60000) ).toMatchSnapshot(); }); it('should work with percentiles', () => { @@ -83,7 +100,7 @@ describe('convertHistogramBucketsToTimeseies(keys, options, buckets)', () => { return { ...bucket, metric_0: { values: { '95.0': 3 } } }; }); expect( - convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithPercentiles) + convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithPercentiles, 60000) ).toMatchSnapshot(); }); it('should throw error with multiple percentiles', () => { @@ -91,7 +108,12 @@ describe('convertHistogramBucketsToTimeseies(keys, options, buckets)', () => { return { ...bucket, metric_0: { values: { '95.0': 3, '99.0': 4 } } }; }); expect(() => - convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithMultiplePercentiles) + convertHistogramBucketsToTimeseries( + keys, + { ...options }, + bucketsWithMultiplePercentiles, + 60000 + ) ).toThrow(); }); it('should work with keyed percentiles', () => { @@ -99,7 +121,7 @@ describe('convertHistogramBucketsToTimeseies(keys, options, buckets)', () => { return { ...bucket, metric_0: { values: [{ key: '99.0', value: 4 }] } }; }); expect( - convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithKeyedPercentiles) + convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithKeyedPercentiles, 60000) ).toMatchSnapshot(); }); it('should throw error with multiple keyed percentiles', () => { @@ -115,11 +137,16 @@ describe('convertHistogramBucketsToTimeseies(keys, options, buckets)', () => { }; }); expect(() => - convertHistogramBucketsToTimeseries(keys, { ...options }, bucketsWithMultipleKeyedPercentiles) + convertHistogramBucketsToTimeseries( + keys, + { ...options }, + bucketsWithMultipleKeyedPercentiles, + 60000 + ) ).toThrow(); }); - it('should tranform top_metric aggregations', () => { + it('should transform top_metric aggregations', () => { const topMetricOptions: MetricsAPIRequest = { ...options, metrics: [ @@ -167,7 +194,7 @@ describe('convertHistogramBucketsToTimeseies(keys, options, buckets)', () => { ]; expect( - convertHistogramBucketsToTimeseries(keys, topMetricOptions, bucketsWithTopAggregation) + convertHistogramBucketsToTimeseries(keys, topMetricOptions, bucketsWithTopAggregation, 60000) ).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts b/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts index ed1ffe4e364d2e..f6761f72fabb92 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts +++ b/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts @@ -64,6 +64,10 @@ const getValue = (valueObject: ValueObjectType) => { return null; }; +const dropOutOfBoundsBuckets = (from: number, to: number, bucketSizeInMillis: number) => ( + row: MetricsAPIRow +) => row.timestamp >= from && row.timestamp + bucketSizeInMillis <= to; + const convertBucketsToRows = ( options: MetricsAPIRequest, buckets: HistogramBucket[] @@ -81,7 +85,8 @@ const convertBucketsToRows = ( export const convertHistogramBucketsToTimeseries = ( keys: string[], options: MetricsAPIRequest, - buckets: HistogramBucket[] + buckets: HistogramBucket[], + bucketSizeInMillis: number ): MetricsAPISeries => { const id = keys.join(':'); // If there are no metrics then we just return the empty series @@ -94,7 +99,11 @@ export const convertHistogramBucketsToTimeseries = ( type: 'number', })) as MetricsAPIColumn[]; const allRows = convertBucketsToRows(options, buckets); - const rows = options.dropLastBucket ? allRows.slice(0, allRows.length - 1) : allRows; + const rows = options.dropPartialBuckets + ? allRows.filter( + dropOutOfBoundsBuckets(options.timerange.from, options.timerange.to, bucketSizeInMillis) + ) + : allRows; return { id, keys, diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts index 0796433c29018c..539e9a1fee6ef0 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts @@ -28,7 +28,7 @@ const BASE_METRICS_UI_OPTIONS: MetricsAPIRequest = { interval: '1m', }, limit: 9, - dropLastBucket: true, + dropPartialBuckets: true, indexPattern: 'metrics-*', metrics: [ { id: 'metric_0', aggregations: { metric_0: { avg: { field: 'system.cpu.user.pct' } } } }, diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts index 6815dd31b2bfd3..b1a19bc6a8d322 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts @@ -26,7 +26,7 @@ export const convertRequestToMetricsAPIOptions = ( indexPattern, limit, metrics, - dropLastBucket: true, + dropPartialBuckets: true, }; if (options.afterKey) { diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.test.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.test.ts index de44cf016886fa..b4e6983a099004 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.test.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.test.ts @@ -114,5 +114,6 @@ const metricsApiRequest = { ], limit: 3000, alignDataToEnd: true, + dropPartialBuckets: true, groupBy: ['kubernetes.pod.uid'], }; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.ts index 811b0da952456c..3901c8677ae9bf 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.ts @@ -47,6 +47,7 @@ export const transformRequestToMetricsAPIRequest = async ({ ? snapshotRequest.overrideCompositeSize : compositeSize, alignDataToEnd: true, + dropPartialBuckets: true, }; const filters = []; diff --git a/x-pack/test/api_integration/apis/metrics_ui/metrics_alerting.ts b/x-pack/test/api_integration/apis/metrics_ui/metrics_alerting.ts index 7cf1019a0a325b..6f9c0f5fc708f0 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/metrics_alerting.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/metrics_alerting.ts @@ -6,11 +6,11 @@ */ import expect from '@kbn/expect'; +import moment from 'moment'; import { getElasticsearchMetricQuery } from '../../../../plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query'; import { MetricExpressionParams } from '../../../../plugins/infra/server/lib/alerting/metric_threshold/types'; import { FtrProviderContext } from '../../ftr_provider_context'; - export default function ({ getService }: FtrProviderContext) { const client = getService('legacyEs'); const index = 'test-index'; @@ -33,7 +33,15 @@ export default function ({ getService }: FtrProviderContext) { describe('querying the entire infrastructure', () => { for (const aggType of aggs) { it(`should work with the ${aggType} aggregator`, async () => { - const searchBody = getElasticsearchMetricQuery(getSearchParams(aggType), '@timestamp'); + const timeframe = { + start: moment().subtract(25, 'minutes').valueOf(), + end: moment().valueOf(), + }; + const searchBody = getElasticsearchMetricQuery( + getSearchParams(aggType), + '@timestamp', + timeframe + ); const result = await client.search({ index, body: searchBody, @@ -44,9 +52,14 @@ export default function ({ getService }: FtrProviderContext) { }); } it('should work with a filterQuery', async () => { + const timeframe = { + start: moment().subtract(25, 'minutes').valueOf(), + end: moment().valueOf(), + }; const searchBody = getElasticsearchMetricQuery( getSearchParams('avg'), '@timestamp', + timeframe, undefined, '{"bool":{"should":[{"match_phrase":{"agent.hostname":"foo"}}],"minimum_should_match":1}}' ); @@ -62,9 +75,14 @@ export default function ({ getService }: FtrProviderContext) { describe('querying with a groupBy parameter', () => { for (const aggType of aggs) { it(`should work with the ${aggType} aggregator`, async () => { + const timeframe = { + start: moment().subtract(25, 'minutes').valueOf(), + end: moment().valueOf(), + }; const searchBody = getElasticsearchMetricQuery( getSearchParams(aggType), '@timestamp', + timeframe, 'agent.id' ); const result = await client.search({ @@ -77,9 +95,14 @@ export default function ({ getService }: FtrProviderContext) { }); } it('should work with a filterQuery', async () => { + const timeframe = { + start: moment().subtract(25, 'minutes').valueOf(), + end: moment().valueOf(), + }; const searchBody = getElasticsearchMetricQuery( getSearchParams('avg'), '@timestamp', + timeframe, 'agent.id', '{"bool":{"should":[{"match_phrase":{"agent.hostname":"foo"}}],"minimum_should_match":1}}' ); diff --git a/x-pack/test/api_integration/apis/metrics_ui/metrics_explorer.ts b/x-pack/test/api_integration/apis/metrics_ui/metrics_explorer.ts index 8d5c677f56ea2f..1eaae741f7e05b 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/metrics_explorer.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/metrics_explorer.ts @@ -57,12 +57,7 @@ export default function ({ getService }: FtrProviderContext) { { name: 'metric_0', type: 'number' }, { name: 'metric_1', type: 'number' }, ]); - expect(firstSeries.rows).to.have.length(9); - expect(firstSeries.rows![1]).to.eql({ - metric_0: 0.005333333333333333, - metric_1: 131, - timestamp: 1547571300000, - }); + expect(firstSeries.rows).to.have.length(8); }); it('should apply filterQuery to data', async () => { @@ -96,11 +91,7 @@ export default function ({ getService }: FtrProviderContext) { { name: 'timestamp', type: 'date' }, { name: 'metric_0', type: 'number' }, ]); - expect(firstSeries.rows).to.have.length(9); - expect(firstSeries.rows![1]).to.eql({ - metric_0: 0.024, - timestamp: 1547571300000, - }); + expect(firstSeries.rows).to.have.length(8); }); it('should work for empty metrics', async () => { @@ -159,12 +150,7 @@ export default function ({ getService }: FtrProviderContext) { { name: 'metric_0', type: 'number' }, { name: 'groupBy', type: 'string' }, ]); - expect(firstSeries.rows).to.have.length(9); - expect(firstSeries.rows![1]).to.eql({ - groupBy: 'system.diskio', - metric_0: 24, - timestamp: 1547571300000, - }); + expect(firstSeries.rows).to.have.length(8); expect(body.pageInfo).to.eql({ afterKey: { groupBy0: 'system.fsstat' }, total: 12, @@ -204,12 +190,7 @@ export default function ({ getService }: FtrProviderContext) { { name: 'metric_0', type: 'number' }, { name: 'groupBy', type: 'string' }, ]); - expect(firstSeries.rows).to.have.length(9); - expect(firstSeries.rows![1]).to.eql({ - groupBy: 'demo-stack-mysql-01 / eth0', - metric_0: 53577.683333333334, - timestamp: 1547571300000, - }); + expect(firstSeries.rows).to.have.length(8); expect(body.pageInfo).to.eql({ afterKey: { groupBy0: 'demo-stack-mysql-01', groupBy1: 'eth2' }, total: 4, diff --git a/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts b/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts index 8fd8a5a6d3ec8e..4181344f064b9b 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts @@ -187,7 +187,7 @@ export default function ({ getService }: FtrProviderContext) { name: 'cpu', value: 0.0032, max: 0.0038333333333333336, - avg: 0.002794444444444445, + avg: 0.003341666666666667, }, ]); } @@ -222,7 +222,7 @@ export default function ({ getService }: FtrProviderContext) { expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01'); expect(firstNode).to.have.property('metrics'); expect(firstNode.metrics[0]).to.have.property('timeseries'); - expect(firstNode.metrics[0].timeseries?.rows.length).to.equal(58); + expect(firstNode.metrics[0].timeseries?.rows.length).to.equal(56); const rows = firstNode.metrics[0].timeseries?.rows; const rowInterval = (rows?.[1]?.timestamp || 0) - (rows?.[0]?.timestamp || 0); expect(rowInterval).to.equal(10000); @@ -257,7 +257,7 @@ export default function ({ getService }: FtrProviderContext) { expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01'); expect(firstNode).to.have.property('metrics'); expect(firstNode.metrics[0]).to.have.property('timeseries'); - expect(firstNode.metrics[0].timeseries?.rows.length).to.equal(7); + expect(firstNode.metrics[0].timeseries?.rows.length).to.equal(5); } }); }); @@ -298,7 +298,7 @@ export default function ({ getService }: FtrProviderContext) { name: 'custom_0', value: 0.0016, max: 0.0018333333333333333, - avg: 0.0013666666666666669, + avg: 0.00165, }, ]); } @@ -389,7 +389,7 @@ export default function ({ getService }: FtrProviderContext) { name: 'cpu', value: 0.0032, max: 0.0038333333333333336, - avg: 0.002794444444444445, + avg: 0.003341666666666667, }, ]); const secondNode = nodes[1] as any; @@ -403,7 +403,7 @@ export default function ({ getService }: FtrProviderContext) { name: 'cpu', value: 0.0032, max: 0.0038333333333333336, - avg: 0.002794444444444445, + avg: 0.003341666666666667, }, ]); } From 75c573a368cd97ec8a565c67111d02e7049d455d Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Mon, 19 Jul 2021 20:19:05 +0100 Subject: [PATCH 09/39] EP Meta Telemetry Perf (#104396) * Add comments for other developers. * Move OS infomation into meta key. * Refmt endpoint metrics. * Add helper funcs to batch sending. * Add test to ensure opt in status. * Add helpers test. * Finish reshaping the document based on feedback. * Add better type safety. Add policy package version to output. * Fix sender implementation for aggregating EP datastreams. * Fix type issues. * Fix cadence inference + miss default agent id. * Dynamically control search ranges for metrics + policy responses. * Set back to 24h. * Add comment for ignoring the default policy id. * explicitly type the sub agg search query. * Improve type safety. * Add additional type safety + try/catch the last block. * Remove unneeded optional chaining. * Destructure host metrics. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/lib/telemetry/diagnostic_task.ts | 4 +- .../lib/telemetry/endpoint_task.test.ts | 34 ++- .../server/lib/telemetry/endpoint_task.ts | 248 +++++++++++++----- .../server/lib/telemetry/helpers.task.ts | 37 --- .../server/lib/telemetry/helpers.test.ts | 127 +++++++++ .../server/lib/telemetry/helpers.ts | 64 ++++- .../server/lib/telemetry/mocks.ts | 17 ++ .../server/lib/telemetry/sender.ts | 36 ++- .../server/lib/telemetry/types.ts | 26 +- 9 files changed, 458 insertions(+), 135 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/helpers.task.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts index 05d7396031a5f9..c83f37593a0363 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts @@ -12,7 +12,7 @@ import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../task_manager/server'; -import { getLastTaskExecutionTimestamp } from './helpers'; +import { getPreviousDiagTaskTimestamp } from './helpers'; import { TelemetryEventsSender, TelemetryEvent } from './sender'; export const TelemetryDiagTaskConstants = { @@ -44,7 +44,7 @@ export class TelemetryDiagTask { return { run: async () => { const executeTo = moment().utc().toISOString(); - const executeFrom = getLastTaskExecutionTimestamp( + const executeFrom = getPreviousDiagTaskTimestamp( executeTo, taskInstance.state?.lastExecutionTimestamp ); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts index a056ef783f6cf6..48c996d1e9eff4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts @@ -6,9 +6,11 @@ */ import { loggingSystemMock } from 'src/core/server/mocks'; +import { TaskStatus } from '../../../../task_manager/server'; import { taskManagerMock } from '../../../../task_manager/server/mocks'; -import { TelemetryEndpointTask } from './endpoint_task'; -import { createMockTelemetryEventsSender } from './mocks'; + +import { TelemetryEndpointTask, TelemetryEndpointTaskConstants } from './endpoint_task'; +import { createMockTelemetryEventsSender, MockTelemetryEndpointTask } from './mocks'; describe('test', () => { let logger: ReturnType; @@ -18,7 +20,7 @@ describe('test', () => { }); describe('endpoint alert telemetry checks', () => { - test('the task can register', () => { + test('the endpoint task can register', () => { const telemetryEndpointTask = new TelemetryEndpointTask( logger, taskManagerMock.createSetup(), @@ -48,4 +50,30 @@ describe('test', () => { await telemetryEndpointTask.start(mockTaskManagerStart); expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); }); + + test('endpoint task should not query elastic if telemetry is not opted in', async () => { + const mockSender = createMockTelemetryEventsSender(false); + const mockTaskManager = taskManagerMock.createSetup(); + new MockTelemetryEndpointTask(logger, mockTaskManager, mockSender); + + const mockTaskInstance = { + id: TelemetryEndpointTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: TelemetryEndpointTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][TelemetryEndpointTaskConstants.TYPE] + .createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(mockSender.fetchDiagnosticAlerts).not.toHaveBeenCalled(); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts index cac92983b38783..71a105d1e58f5c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts @@ -12,23 +12,43 @@ import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../task_manager/server'; -import { getLastTaskExecutionTimestamp } from './helpers'; +import { + batchTelemetryRecords, + getPreviousEpMetaTaskTimestamp, + isPackagePolicyList, +} from './helpers'; import { TelemetryEventsSender } from './sender'; -import { FullAgentPolicyInput } from '../../../../fleet/common/types/models/agent_policy'; +import { PolicyData } from '../../../common/endpoint/types'; +import { FLEET_ENDPOINT_PACKAGE } from '../../../../fleet/common'; import { EndpointMetricsAggregation, EndpointPolicyResponseAggregation, EndpointPolicyResponseDocument, - FleetAgentCacheItem, } from './types'; export const TelemetryEndpointTaskConstants = { TIMEOUT: '5m', TYPE: 'security:endpoint-meta-telemetry', - INTERVAL: '24m', + INTERVAL: '24h', VERSION: '1.0.0', }; +// Endpoint agent uses this Policy ID while it's installing. +const DefaultEndpointPolicyIdToIgnore = '00000000-0000-0000-0000-000000000000'; + +const EmptyFleetAgentResponse = { + agents: [], + total: 0, + page: 0, + perPage: 0, +}; + +/** Telemetry Endpoint Task + * + * The Endpoint Telemetry task is a daily batch job that collects and transmits non-sensitive + * endpoint performance and policy logs to Elastic Security Data Engineering. It is used to + * identify bugs or common UX issues with the Elastic Security Endpoint agent. + */ export class TelemetryEndpointTask { private readonly logger: Logger; private readonly sender: TelemetryEventsSender; @@ -50,17 +70,21 @@ export class TelemetryEndpointTask { return { run: async () => { - const executeTo = moment().utc().toISOString(); - const lastExecutionTimestamp = getLastTaskExecutionTimestamp( - executeTo, + const taskExecutionTime = moment().utc().toISOString(); + const lastExecutionTimestamp = getPreviousEpMetaTaskTimestamp( + taskExecutionTime, taskInstance.state?.lastExecutionTimestamp ); - const hits = await this.runTask(taskInstance.id); + const hits = await this.runTask( + taskInstance.id, + lastExecutionTimestamp, + taskExecutionTime + ); return { state: { - lastExecutionTimestamp, + lastExecutionTimestamp: taskExecutionTime, runs: (state.runs || 0) + 1, hits, }, @@ -94,23 +118,25 @@ export class TelemetryEndpointTask { return `${TelemetryEndpointTaskConstants.TYPE}:${TelemetryEndpointTaskConstants.VERSION}`; }; - private async fetchEndpointData() { - const [epMetricsResponse, fleetAgentsResponse, policyResponse] = await Promise.allSettled([ - this.sender.fetchEndpointMetrics(), + private async fetchEndpointData(executeFrom: string, executeTo: string) { + const [fleetAgentsResponse, epMetricsResponse, policyResponse] = await Promise.allSettled([ this.sender.fetchFleetAgents(), - this.sender.fetchFailedEndpointPolicyResponses(), + this.sender.fetchEndpointMetrics(executeFrom, executeTo), + this.sender.fetchEndpointPolicyResponses(executeFrom, executeTo), ]); return { + fleetAgentsResponse: + fleetAgentsResponse.status === 'fulfilled' + ? fleetAgentsResponse.value + : EmptyFleetAgentResponse, endpointMetrics: epMetricsResponse.status === 'fulfilled' ? epMetricsResponse.value : undefined, - fleetAgentsResponse: - fleetAgentsResponse.status === 'fulfilled' ? fleetAgentsResponse.value : undefined, epPolicyResponse: policyResponse.status === 'fulfilled' ? policyResponse.value : undefined, }; } - public runTask = async (taskId: string) => { + public runTask = async (taskId: string, executeFrom: string, executeTo: string) => { if (taskId !== this.getTaskId()) { this.logger.debug(`Outdated task running: ${taskId}`); return 0; @@ -122,99 +148,179 @@ export class TelemetryEndpointTask { return 0; } - const endpointData = await this.fetchEndpointData(); + const endpointData = await this.fetchEndpointData(executeFrom, executeTo); + + /** STAGE 1 - Fetch Endpoint Agent Metrics + * + * Reads Endpoint Agent metrics out of the `.ds-metrics-endpoint.metrics` data stream + * and buckets them by Endpoint Agent id and sorts by the top hit. The EP agent will + * report its metrics once per day OR every time a policy change has occured. If + * a metric document(s) exists for an EP agent we map to fleet agent and policy + */ + if (endpointData.endpointMetrics === undefined) { + this.logger.debug(`no endpoint metrics to report`); + return 0; + } const { body: endpointMetricsResponse } = (endpointData.endpointMetrics as unknown) as { body: EndpointMetricsAggregation; }; - if (endpointMetricsResponse.aggregations === undefined) { - this.logger.debug(`No endpoint metrics`); - return 0; - } const endpointMetrics = endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( (epMetrics) => { return { endpoint_agent: epMetrics.latest_metrics.hits.hits[0]._source.agent.id, + endpoint_version: epMetrics.latest_metrics.hits.hits[0]._source.agent.version, endpoint_metrics: epMetrics.latest_metrics.hits.hits[0]._source, }; } ); - if (endpointMetrics.length === 0) { - this.logger.debug('no reported endpoint metrics'); - return 0; - } - + /** STAGE 2 - Fetch Fleet Agent Config + * + * As the policy id + policy version does not exist on the Endpoint Metrics document + * we need to fetch information about the Fleet Agent and sync the metrics document + * with the Fleet agent's policy data. + * + * 7.14 ~ An issue was created with the Endpoint agent team to add the policy id + + * policy version to the metrics document to circumvent and refactor away from + * this expensive join operation. + */ const agentsResponse = endpointData.fleetAgentsResponse; if (agentsResponse === undefined) { - this.logger.debug('no agents to report'); return 0; } + const fleetAgents = agentsResponse.agents.reduce((cache, agent) => { + if (agent.id === DefaultEndpointPolicyIdToIgnore) { + return cache; + } + + if (agent.policy_id !== null && agent.policy_id !== undefined) { + cache.set(agent.id, agent.policy_id); + } - const fleetAgents = agentsResponse?.agents.reduce((cache, agent) => { - cache.set(agent.id, { policy_id: agent.policy_id, policy_version: agent.policy_revision }); return cache; - }, new Map()); + }, new Map()); - const endpointPolicyCache = new Map(); + const endpointPolicyCache = new Map(); for (const policyInfo of fleetAgents.values()) { - if ( - policyInfo.policy_id !== null && - policyInfo.policy_id !== undefined && - !endpointPolicyCache.has(policyInfo.policy_id) - ) { - const packagePolicies = await this.sender.fetchEndpointPolicyConfigs(policyInfo.policy_id); - packagePolicies?.inputs.forEach((input) => { - if (input.type === 'endpoint' && policyInfo.policy_id !== undefined) { - endpointPolicyCache.set(policyInfo.policy_id, input); - } - }); + if (policyInfo !== null && policyInfo !== undefined && !endpointPolicyCache.has(policyInfo)) { + const agentPolicy = await this.sender.fetchPolicyConfigs(policyInfo); + const packagePolicies = agentPolicy?.package_policies; + + if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { + packagePolicies + .map((pPolicy) => pPolicy as PolicyData) + .forEach((pPolicy) => { + if (pPolicy.inputs[0].config !== undefined) { + pPolicy.inputs.forEach((input) => { + if ( + input.type === FLEET_ENDPOINT_PACKAGE && + input.config !== undefined && + policyInfo !== undefined + ) { + endpointPolicyCache.set(policyInfo, pPolicy); + } + }); + } + }); + } } } + /** STAGE 3 - Fetch Endpoint Policy Responses + * + * Reads Endpoint Agent policy responses out of the `.ds-metrics-endpoint.policy*` data + * stream and creates a local K/V structure that stores the policy response (V) with + * the Endpoint Agent Id (K). A value will only exist if there has been a endpoint + * enrolled in the last 24 hours OR a policy change has occurred. We only send + * non-successful responses. If the field is null, we assume no responses in + * the last 24h or no failures/warnings in the policy applied. + * + */ const { body: failedPolicyResponses } = (endpointData.epPolicyResponse as unknown) as { body: EndpointPolicyResponseAggregation; }; const policyResponses = failedPolicyResponses.aggregations.policy_responses.buckets.reduce( - (cache, bucket) => { - const doc = bucket.latest_response.hits.hits[0]; - cache.set(bucket.key, doc); + (cache, endpointAgentId) => { + const doc = endpointAgentId.latest_response.hits.hits[0]; + cache.set(endpointAgentId.key, doc); return cache; }, new Map() ); - const telemetryPayloads = endpointMetrics.map((endpoint) => { - let policyConfig = null; - let failedPolicy = null; + /** STAGE 4 - Create the telemetry log records + * + * Iterates through the endpoint metrics documents at STAGE 1 and joins them together + * to form the telemetry log that is sent back to Elastic Security developers to + * make improvements to the product. + * + */ + try { + const telemetryPayloads = endpointMetrics.map((endpoint) => { + let policyConfig = null; + let failedPolicy = null; + + const fleetAgentId = endpoint.endpoint_metrics.elastic.agent.id; + const endpointAgentId = endpoint.endpoint_agent; - const fleetAgentId = endpoint.endpoint_metrics.elastic.agent.id; - const endpointAgentId = endpoint.endpoint_agent; + const policyInformation = fleetAgents.get(fleetAgentId); + if (policyInformation) { + policyConfig = endpointPolicyCache.get(policyInformation); - const policyInformation = fleetAgents.get(fleetAgentId); - if (policyInformation?.policy_id) { - policyConfig = endpointPolicyCache.get(policyInformation?.policy_id); - if (policyConfig) { - failedPolicy = policyResponses.get(policyConfig?.id); + if (policyConfig) { + failedPolicy = policyResponses.get(policyConfig?.id); + } } - } - return { - agent_id: fleetAgentId, - endpoint_id: endpointAgentId, - endpoint_metrics: { - os: endpoint.endpoint_metrics.host.os, - cpu: endpoint.endpoint_metrics.Endpoint.metrics.cpu, - memory: endpoint.endpoint_metrics.Endpoint.metrics.memory, - uptime: endpoint.endpoint_metrics.Endpoint.metrics.uptime, - }, - policy_config: policyConfig, - policy_failure: failedPolicy, - }; - }); + const { cpu, memory, uptime } = endpoint.endpoint_metrics.Endpoint.metrics; + + return { + '@timestamp': executeTo, + agent_id: fleetAgentId, + endpoint_id: endpointAgentId, + endpoint_version: endpoint.endpoint_version, + endpoint_package_version: policyConfig?.package?.version || null, + endpoint_metrics: { + cpu: cpu.endpoint, + memory: memory.endpoint.private, + uptime, + }, + endpoint_meta: { + os: endpoint.endpoint_metrics.host.os, + }, + policy_config: policyConfig !== null ? policyConfig?.inputs[0].config.policy : {}, + policy_response: + failedPolicy !== null && failedPolicy !== undefined + ? { + agent_policy_status: failedPolicy._source.event.agent_id_status, + manifest_version: + failedPolicy._source.Endpoint.policy.applied.artifacts.global.version, + status: failedPolicy._source.Endpoint.policy.applied.status, + actions: failedPolicy._source.Endpoint.policy.applied.actions + .map((action) => (action.status !== 'success' ? action : null)) + .filter((action) => action !== null), + } + : {}, + telemetry_meta: { + metrics_timestamp: endpoint.endpoint_metrics['@timestamp'], + }, + }; + }); - this.sender.sendOnDemand('endpoint-metadata', telemetryPayloads); - return telemetryPayloads.length; + /** + * STAGE 5 - Send the documents + * + * Send the documents in a batches of 100 + */ + batchTelemetryRecords(telemetryPayloads, 100).forEach((telemetryBatch) => + this.sender.sendOnDemand('endpoint-metadata', telemetryBatch) + ); + return telemetryPayloads.length; + } catch (err) { + this.logger.error('Could not send endpoint alert telemetry'); + return 0; + } }; } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.task.ts deleted file mode 100644 index ec81f3d0a5fa4b..00000000000000 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.task.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import moment from 'moment'; -import { getLastTaskExecutionTimestamp } from './helpers'; - -describe('test scheduled task helpers', () => { - test('test -5 mins is returned when there is no previous task run', async () => { - const executeTo = moment().utc().toISOString(); - const executeFrom = undefined; - const newExecuteFrom = getLastTaskExecutionTimestamp(executeTo, executeFrom); - - expect(newExecuteFrom).toEqual(moment(executeTo).subtract(5, 'minutes').toISOString()); - }); - - test('test -6 mins is returned when there was a previous task run', async () => { - const executeTo = moment().utc().toISOString(); - const executeFrom = moment(executeTo).subtract(6, 'minutes').toISOString(); - const newExecuteFrom = getLastTaskExecutionTimestamp(executeTo, executeFrom); - - expect(newExecuteFrom).toEqual(executeFrom); - }); - - // it's possible if Kibana is down for a prolonged period the stored lastRun would have drifted - // if that is the case we will just roll it back to a 10 min search window - test('test 10 mins is returned when previous task run took longer than 10 minutes', async () => { - const executeTo = moment().utc().toISOString(); - const executeFrom = moment(executeTo).subtract(142, 'minutes').toISOString(); - const newExecuteFrom = getLastTaskExecutionTimestamp(executeTo, executeFrom); - - expect(newExecuteFrom).toEqual(moment(executeTo).subtract(10, 'minutes').toISOString()); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts new file mode 100644 index 00000000000000..bee673fc8725f7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { createMockPackagePolicy } from './mocks'; +import { + getPreviousDiagTaskTimestamp, + getPreviousEpMetaTaskTimestamp, + batchTelemetryRecords, + isPackagePolicyList, +} from './helpers'; + +describe('test diagnostic telemetry scheduled task timing helper', () => { + test('test -5 mins is returned when there is no previous task run', async () => { + const executeTo = moment().utc().toISOString(); + const executeFrom = undefined; + const newExecuteFrom = getPreviousDiagTaskTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(5, 'minutes').toISOString()); + }); + + test('test -6 mins is returned when there was a previous task run', async () => { + const executeTo = moment().utc().toISOString(); + const executeFrom = moment(executeTo).subtract(6, 'minutes').toISOString(); + const newExecuteFrom = getPreviousDiagTaskTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(executeFrom); + }); + + // it's possible if Kibana is down for a prolonged period the stored lastRun would have drifted + // if that is the case we will just roll it back to a 10 min search window + test('test 10 mins is returned when previous task run took longer than 10 minutes', async () => { + const executeTo = moment().utc().toISOString(); + const executeFrom = moment(executeTo).subtract(142, 'minutes').toISOString(); + const newExecuteFrom = getPreviousDiagTaskTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(10, 'minutes').toISOString()); + }); +}); + +describe('test endpoint meta telemetry scheduled task timing helper', () => { + test('test -24 hours is returned when there is no previous task run', async () => { + const executeTo = moment().utc().toISOString(); + const executeFrom = undefined; + const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(24, 'hours').toISOString()); + }); + + test('test -24 hours is returned when there was a previous task run', async () => { + const executeTo = moment().utc().toISOString(); + const executeFrom = moment(executeTo).subtract(24, 'hours').toISOString(); + const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(executeFrom); + }); + + // it's possible if Kibana is down for a prolonged period the stored lastRun would have drifted + // if that is the case we will just roll it back to a 30 hour search window + test('test 24 hours is returned when previous task run took longer than 24 hours', async () => { + const executeTo = moment().utc().toISOString(); + const executeFrom = moment(executeTo).subtract(72, 'hours').toISOString(); // down 3 days + const newExecuteFrom = getPreviousEpMetaTaskTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(24, 'hours').toISOString()); + }); +}); + +describe('telemetry batching logic', () => { + test('records can be batched oddly as they are sent to the telemetry channel', async () => { + const stubTelemetryRecords = [...Array(10).keys()]; + const batchSize = 3; + + const records = batchTelemetryRecords(stubTelemetryRecords, batchSize); + expect(records.length).toEqual(4); + }); + + test('records can be batched evenly as they are sent to the telemetry channel', async () => { + const stubTelemetryRecords = [...Array(299).keys()]; + const batchSize = 100; + + const records = batchTelemetryRecords(stubTelemetryRecords, batchSize); + expect(records.length).toEqual(3); + }); + + test('empty telemetry records wont be batched', async () => { + const stubTelemetryRecords = [...Array(0).keys()]; + const batchSize = 100; + + const records = batchTelemetryRecords(stubTelemetryRecords, batchSize); + expect(records.length).toEqual(0); + }); +}); + +describe('test package policy type guard', () => { + test('string records are not package policies', async () => { + const arr = ['a', 'b', 'c']; + const result = isPackagePolicyList(arr); + + expect(result).toEqual(false); + }); + + test('empty array are not package policies', () => { + const arr: string[] = []; + const result = isPackagePolicyList(arr); + + expect(result).toEqual(false); + }); + + test('undefined is not a list of package policies', () => { + const arr = undefined; + const result = isPackagePolicyList(arr); + + expect(result).toEqual(false); + }); + + test('package policies are list of package policies', () => { + const arr = [createMockPackagePolicy(), createMockPackagePolicy(), createMockPackagePolicy()]; + const result = isPackagePolicyList(arr); + + expect(result).toEqual(true); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index e820116462fa20..b32bd64a073378 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -6,8 +6,16 @@ */ import moment from 'moment'; +import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; -export const getLastTaskExecutionTimestamp = ( +/** + * Determines the when the last run was in order to execute to. + * + * @param executeTo + * @param lastExecutionTimestamp + * @returns the timestamp to search from + */ +export const getPreviousDiagTaskTimestamp = ( executeTo: string, lastExecutionTimestamp?: string ) => { @@ -21,3 +29,57 @@ export const getLastTaskExecutionTimestamp = ( return lastExecutionTimestamp; }; + +/** + * Determines the when the last run was in order to execute to. + * + * @param executeTo + * @param lastExecutionTimestamp + * @returns the timestamp to search from + */ +export const getPreviousEpMetaTaskTimestamp = ( + executeTo: string, + lastExecutionTimestamp?: string +) => { + if (lastExecutionTimestamp === undefined) { + return moment(executeTo).subtract(24, 'hours').toISOString(); + } + + if (moment(executeTo).diff(lastExecutionTimestamp, 'hours') >= 24) { + return moment(executeTo).subtract(24, 'hours').toISOString(); + } + + return lastExecutionTimestamp; +}; + +/** + * Chunks an Array into an Array> + * This is to prevent overloading the telemetry channel + user resources + * + * @param telemetryRecords + * @param batchSize + * @returns the batch of records + */ +export const batchTelemetryRecords = ( + telemetryRecords: unknown[], + batchSize: number +): unknown[][] => + [...Array(Math.ceil(telemetryRecords.length / batchSize))].map((_) => + telemetryRecords.splice(0, batchSize) + ); + +/** + * User defined type guard for PackagePolicy + * + * @param data the union type of package policies + * @returns type confirmation + */ +export function isPackagePolicyList( + data: string[] | PackagePolicy[] | undefined +): data is PackagePolicy[] { + if (data === undefined || data.length < 1) { + return false; + } + + return (data as PackagePolicy[])[0].inputs !== undefined; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts index 6738113da103d5..f27d22287c9d7c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts @@ -9,6 +9,7 @@ import { TelemetryEventsSender } from './sender'; import { TelemetryDiagTask } from './diagnostic_task'; import { TelemetryEndpointTask } from './endpoint_task'; +import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; /** * Creates a mocked Telemetry Events Sender @@ -33,6 +34,22 @@ export const createMockTelemetryEventsSender = ( } as unknown) as jest.Mocked; }; +/** + * Creates a mocked package policy + */ +export const createMockPackagePolicy = (): jest.Mocked => { + return ({ + id: jest.fn(), + inputs: jest.fn(), + version: jest.fn(), + revision: jest.fn(), + updated_at: jest.fn(), + updated_by: jest.fn(), + created_at: jest.fn(), + created_by: jest.fn(), + } as unknown) as jest.Mocked; +}; + /** * Creates a mocked Telemetry Diagnostic Task */ diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 302f56802a5a49..6f9279d04b3480 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -7,6 +7,7 @@ import { cloneDeep } from 'lodash'; import axios from 'axios'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { LegacyAPICaller, SavedObjectsClientContract } from 'kibana/server'; import { URL } from 'url'; import { CoreStart, ElasticsearchClient, Logger } from 'src/core/server'; @@ -139,22 +140,30 @@ export class TelemetryEventsSender { return callCluster('search', query); } - public async fetchEndpointMetrics() { + public async fetchEndpointMetrics(executeFrom: string, executeTo: string) { if (this.esClient === undefined) { throw Error('could not fetch policy responses. es client is not available'); } - const query = { + const query: SearchRequest = { expand_wildcards: 'open,hidden', - index: `.ds-metrics-endpoint.metrics*`, + index: `.ds-metrics-endpoint.metrics-*`, ignore_unavailable: false, size: 0, // no query results required - only aggregation quantity body: { + query: { + range: { + '@timestamp': { + gte: executeFrom, + lt: executeTo, + }, + }, + }, aggs: { endpoint_agents: { terms: { + field: 'agent.id', size: this.max_records, - field: 'agent.id.keyword', }, aggs: { latest_metrics: { @@ -175,7 +184,6 @@ export class TelemetryEventsSender { }, }; - // @ts-expect-error The types of 'body.aggs' are incompatible between these types. return this.esClient.search(query); } @@ -192,35 +200,38 @@ export class TelemetryEventsSender { }); } - public async fetchEndpointPolicyConfigs(id: string) { + public async fetchPolicyConfigs(id: string) { if (this.savedObjectClient === undefined) { throw Error('could not fetch endpoint policy configs. saved object client is not available'); } - return this.agentPolicyService?.getFullAgentPolicy(this.savedObjectClient, id); + return this.agentPolicyService?.get(this.savedObjectClient, id); } - public async fetchFailedEndpointPolicyResponses() { + public async fetchEndpointPolicyResponses(executeFrom: string, executeTo: string) { if (this.esClient === undefined) { throw Error('could not fetch policy responses. es client is not available'); } - const query = { + const query: SearchRequest = { expand_wildcards: 'open,hidden', index: `.ds-metrics-endpoint.policy*`, ignore_unavailable: false, size: 0, // no query results required - only aggregation quantity body: { query: { - match: { - 'Endpoint.policy.applied.status': 'failure', + range: { + '@timestamp': { + gte: executeFrom, + lt: executeTo, + }, }, }, aggs: { policy_responses: { terms: { size: this.max_records, - field: 'Endpoint.policy.applied.id.keyword', + field: 'Endpoint.policy.applied.id', }, aggs: { latest_response: { @@ -241,7 +252,6 @@ export class TelemetryEventsSender { }, }; - // @ts-expect-error The types of 'body.aggs' are incompatible between these types. return this.esClient.search(query); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index 435f3cf49d1f1b..355393145fa0b3 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -5,13 +5,6 @@ * 2.0. */ -// Sec Sol Kbn telemetry instrumentation specific - -export interface FleetAgentCacheItem { - policy_id: string | undefined; - policy_version: number | undefined | null; -} - // EP Policy Response export interface EndpointPolicyResponseAggregation { @@ -45,7 +38,23 @@ export interface EndpointPolicyResponseDocument { event: { agent_id_status: string; }; - Endpoint: {}; + Endpoint: { + policy: { + applied: { + actions: Array<{ + name: string; + message: string; + status: string; + }>; + artifacts: { + global: { + version: string; + }; + }; + status: string; + }; + }; + }; }; } @@ -74,6 +83,7 @@ interface EndpointMetricDocument { '@timestamp': string; agent: { id: string; + version: string; }; Endpoint: { metrics: EndpointMetrics; From 15285bf03b43af207e1c6be70776d3f9563e4855 Mon Sep 17 00:00:00 2001 From: Kevin Logan <56395104+kevinlog@users.noreply.github.com> Date: Mon, 19 Jul 2021 15:24:25 -0400 Subject: [PATCH 10/39] [Security Solution] update text for Isolation action submissions (#105956) --- .../components/user_action_tree/translations.ts | 4 ++-- ...user_action_host_isolation_comment_event.test.tsx | 2 +- .../pages/endpoint_hosts/view/translations.ts | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/cases/public/components/user_action_tree/translations.ts b/x-pack/plugins/cases/public/components/user_action_tree/translations.ts index 54738e29060f3d..40d6fc5bc2ad80 100644 --- a/x-pack/plugins/cases/public/components/user_action_tree/translations.ts +++ b/x-pack/plugins/cases/public/components/user_action_tree/translations.ts @@ -58,11 +58,11 @@ export const UNKNOWN_RULE = i18n.translate('xpack.cases.caseView.unknownRule.lab }); export const ISOLATED_HOST = i18n.translate('xpack.cases.caseView.isolatedHost', { - defaultMessage: 'isolated host', + defaultMessage: 'submitted isolate request on host', }); export const RELEASED_HOST = i18n.translate('xpack.cases.caseView.releasedHost', { - defaultMessage: 'released host', + defaultMessage: 'submitted release request on host', }); export const OTHER_ENDPOINTS = (endpoints: number): string => diff --git a/x-pack/plugins/cases/public/components/user_action_tree/user_action_host_isolation_comment_event.test.tsx b/x-pack/plugins/cases/public/components/user_action_tree/user_action_host_isolation_comment_event.test.tsx index 636cd7e40aac12..80f9985ef15c15 100644 --- a/x-pack/plugins/cases/public/components/user_action_tree/user_action_host_isolation_comment_event.test.tsx +++ b/x-pack/plugins/cases/public/components/user_action_tree/user_action_host_isolation_comment_event.test.tsx @@ -26,7 +26,7 @@ describe('UserActionHostIsolationCommentEvent', () => { it('renders with the correct action and hostname', async () => { const wrapper = mount(); expect(wrapper.find(`[data-test-subj="actions-link-e1"]`).first().exists()).toBeTruthy(); - expect(wrapper.text()).toBe('isolated host host1'); + expect(wrapper.text()).toBe('submitted isolate request on host host1'); }); it('navigates to app on link click', async () => { diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts index 7759935aa840af..57ad3e4808bd5a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts @@ -54,13 +54,13 @@ export const ACTIVITY_LOG = { isolatedAction: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.action.isolated', { - defaultMessage: 'isolated host', + defaultMessage: 'submitted request: Isolate host', } ), unisolatedAction: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.action.unisolated', { - defaultMessage: 'released host', + defaultMessage: 'submitted request: Release host', } ), }, @@ -68,25 +68,25 @@ export const ACTIVITY_LOG = { isolationSuccessful: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.isolationSuccessful', { - defaultMessage: 'host isolation successful', + defaultMessage: 'Host isolation request received by Endpoint', } ), isolationFailed: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.isolationFailed', { - defaultMessage: 'host isolation failed', + defaultMessage: 'Host isolation request received by Endpoint with errors', } ), unisolationSuccessful: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.unisolationSuccessful', { - defaultMessage: 'host release successful', + defaultMessage: 'Release host request received by Endpoint', } ), unisolationFailed: i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.unisolationFailed', { - defaultMessage: 'host release failed', + defaultMessage: 'Release host request received by Endpoint with errors', } ), }, From 7e4c73ad2e2788c42b9de9c08abe04cb4ef3775b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ece=20=C3=96zalp?= Date: Mon, 19 Jul 2021 15:36:22 -0400 Subject: [PATCH 11/39] [CTI] Adds indicator match rule improvements (#97310) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../signals/threat_mapping/get_threat_list.test.ts | 10 +++++----- .../signals/threat_mapping/get_threat_list.ts | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.test.ts index 8d301f221b3472..65dc3794123c70 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.test.ts @@ -16,7 +16,7 @@ describe('get_threat_signals', () => { index: ['index-123'], listItemIndex: 'list-index-123', }); - expect(sortOrder).toEqual([{ '@timestamp': 'asc' }]); + expect(sortOrder).toEqual([{ '@timestamp': 'desc' }]); }); test('it should return sort field of just tie_breaker_id if given no sort order for a list item index', () => { @@ -29,14 +29,14 @@ describe('get_threat_signals', () => { expect(sortOrder).toEqual([{ tie_breaker_id: 'asc' }]); }); - test('it should return sort field of timestamp with asc even if sortOrder is changed as it is hard wired in', () => { + test('it should return sort field of timestamp with desc even if sortOrder is changed as it is hard wired in', () => { const sortOrder = getSortWithTieBreaker({ sortField: undefined, sortOrder: 'desc', index: ['index-123'], listItemIndex: 'list-index-123', }); - expect(sortOrder).toEqual([{ '@timestamp': 'asc' }]); + expect(sortOrder).toEqual([{ '@timestamp': 'desc' }]); }); test('it should return sort field of tie_breaker_id with asc even if sortOrder is changed as it is hard wired in for a list item index', () => { @@ -56,7 +56,7 @@ describe('get_threat_signals', () => { index: ['index-123'], listItemIndex: 'list-index-123', }); - expect(sortOrder).toEqual([{ 'some-field': 'asc', '@timestamp': 'asc' }]); + expect(sortOrder).toEqual([{ 'some-field': 'asc', '@timestamp': 'desc' }]); }); test('it should return sort field of an extra field if given one for a list item index', () => { @@ -76,7 +76,7 @@ describe('get_threat_signals', () => { index: ['index-123'], listItemIndex: 'list-index-123', }); - expect(sortOrder).toEqual([{ 'some-field': 'desc', '@timestamp': 'asc' }]); + expect(sortOrder).toEqual([{ 'some-field': 'desc', '@timestamp': 'desc' }]); }); test('it should return sort field of desc if given one for a list item index', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts index 3ff23e27547b46..8fab8f30fb3dca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -70,6 +70,7 @@ export const getThreatList = async ({ listItemIndex: listClient.getListItemIndex(), }), }, + track_total_hits: false, ignore_unavailable: true, index, size: calculatedPerPage, @@ -101,9 +102,9 @@ export const getSortWithTieBreaker = ({ } } else { if (sortField != null) { - return [{ [sortField]: ascOrDesc, '@timestamp': 'asc' }]; + return [{ [sortField]: ascOrDesc, '@timestamp': 'desc' }]; } else { - return [{ '@timestamp': 'asc' }]; + return [{ '@timestamp': 'desc' }]; } } }; From fc49ae12e6f843e4735488b2c79e1ec530e619b9 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Mon, 19 Jul 2021 13:29:18 -0700 Subject: [PATCH 12/39] docs: APM RUM Source map API (#105332) --- docs/apm/api.asciidoc | 242 +++++++++++++++++++++++++++++++- docs/apm/apm-app-users.asciidoc | 57 ++++---- docs/apm/index.asciidoc | 2 + 3 files changed, 269 insertions(+), 32 deletions(-) diff --git a/docs/apm/api.asciidoc b/docs/apm/api.asciidoc index d9a8d0558714f0..fe4c8a9280158b 100644 --- a/docs/apm/api.asciidoc +++ b/docs/apm/api.asciidoc @@ -11,6 +11,7 @@ Some APM app features are provided via a REST API: * <> * <> * <> +* <> [float] [[apm-api-example]] @@ -72,6 +73,7 @@ curl -X POST \ //// ******************************************************* +******************************************************* //// [role="xpack"] @@ -202,11 +204,9 @@ DELETE /api/apm/settings/agent-configuration ******************************************************* //// - [[apm-list-config]] ==== List configuration - [[apm-list-config-req]] ===== Request @@ -274,7 +274,6 @@ GET /api/apm/settings/agent-configuration ******************************************************* //// - [[apm-search-config]] ==== Search configuration @@ -472,6 +471,7 @@ curl -X POST \ //// ******************************************************* +******************************************************* //// [[kibana-api]] @@ -553,3 +553,239 @@ The API returns the following: // More examples will go here More information on Kibana's API is available in <>. + +//// +******************************************************* +******************************************************* +//// + +[role="xpack"] +[[rum-sourcemap-api]] +=== RUM source map API + +IMPORTANT: This endpoint is only compatible with the +{apm-server-ref}/apm-integration.html[APM integration for Elastic Agent]. +Users with a standalone APM Server should instead use the APM Server +{apm-server-ref}/sourcemap-api.html[source map upload API]. + +A source map allows minified files to be mapped back to original source code -- +allowing you to maintain the speed advantage of minified code, +without losing the ability to quickly and easily debug your application. + +For best results, uploading source maps should become a part of your deployment procedure, +and not something you only do when you see unhelpful errors. +That’s because uploading source maps after errors happen won’t make old errors magically readable -- +errors must occur again for source mapping to occur. + +The following APIs are available: + +* <> +* <> +* <> + +[float] +[[use-sourcemap-api]] +==== How to use APM APIs + +.Expand for required headers, privileges, and usage details +[%collapsible%closed] +====== +include::api.asciidoc[tag=using-the-APIs] +====== + +//// +******************************************************* +//// + +[[rum-sourcemap-post]] +==== Create or update source map + +Create or update a source map for a specific service and version. + +[[rum-sourcemap-post-privs]] +===== Privileges + +The user accessing this endpoint requires `All` Kibana privileges for the {beat_kib_app} feature. +For more information, see <>. + +[[apm-sourcemap-post-req]] +===== Request + +`POST /api/apm/sourcemaps` + +[role="child_attributes"] +[[apm-sourcemap-post-req-body]] +===== Request body + +`service_name`:: +(required, string) The name of the service that the service map should apply to. + +`service_version`:: +(required, string) The version of the service that the service map should apply to. + +`bundle_filepath`:: +(required, string) The absolute path of the final bundle as used in the web application. + +`sourcemap`:: +(required, string or file upload) The source map. It must follow the +https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k[source map revision 3 proposal]. + +[[apm-sourcemap-post-example]] +===== Examples + +The following example uploads a source map for a service named `foo` and a service version of `1.0.0`: + +[source,curl] +-------------------------------------------------- +curl -X POST "http://localhost:5601/api/apm/sourcemaps" \ +-H 'Content-Type: multipart/form-data' \ +-H 'kbn-xsrf: true' \ +-H 'Authorization: ApiKey ${YOUR_API_KEY}' \ +-F 'service_name="foo"' \ +-F 'service_version="1.0.0"' \ +-F 'bundle_filepath="/test/e2e/general-usecase/bundle.js.map"' \ +-F 'sourcemap="{\"version\":3,\"file\":\"static/js/main.chunk.js\",\"sources\":[\"fleet-source-map-client/src/index.css\",\"fleet-source-map-client/src/App.js\",\"webpack:///./src/index.css?bb0a\",\"fleet-source-map-client/src/index.js\",\"fleet-source-map-client/src/reportWebVitals.js\"],\"sourcesContent\":[\"content\"],\"mappings\":\"mapping\",\"sourceRoot\":\"\"}"' <1> +-------------------------------------------------- +<1> Alternatively, upload the source map as a file with `-F 'sourcemap=@path/to/source_map/bundle.js.map'` + +[[apm-sourcemap-post-body]] +===== Response body + +[source,js] +-------------------------------------------------- +{ + "type": "sourcemap", + "identifier": "foo-1.0.0", + "relative_url": "/api/fleet/artifacts/foo-1.0.0/644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456", + "body": "eJyFkL1OwzAUhd/Fc+MbYMuCEBIbHRjKgBgc96R16tiWr1OQqr47NwqJxEK3q/PzWccXxchnZ7E1A1SjuhjVZtF2yOxiEPlO17oWox3D3uPFeSRTjmJQARfCPeiAgGx8NTKsYdAc1T3rwaSJGcds8Sp3c1HnhfywUZ3QhMTFFGepZxqMC9oex3CS9tpk1XyozgOlmoVKuJX1DqEQZ0su7PGtLU+V/3JPKc3cL7TJ2FNDRPov4bFta3MDM4f7W69lpJjLO9qdK8bzVPhcJz3HUCQ4LbO/p5hCSC4cZPByrp/wFqOklbpefwAhzpqI", + "created": "2021-07-09T20:47:44.812Z", + "id": "apm:foo-1.0.0-644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456", + "compressionAlgorithm": "zlib", + "decodedSha256": "644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456", + "decodedSize": 441, + "encodedSha256": "024c72749c3e3dd411b103f7040ae62633558608f480bce4b108cf5b2275bd24", + "encodedSize": 237, + "encryptionAlgorithm": "none", + "packageName": "apm" +} +-------------------------------------------------- + +//// +******************************************************* +//// + +[[rum-sourcemap-get]] +==== Get source maps + +Returns an array of Fleet artifacts, including source map uploads. + +[[rum-sourcemap-get-privs]] +===== Privileges + +The user accessing this endpoint requires `Read` or `All` Kibana privileges for the {beat_kib_app} feature. +For more information, see <>. + +[[apm-sourcemap-get-req]] +===== Request + +`GET /api/apm/sourcemaps` + +[[apm-sourcemap-get-example]] +===== Example + +The following example requests all uploaded source maps: + +[source,curl] +-------------------------------------------------- +curl -X GET "http://localhost:5601/api/apm/sourcemaps" \ +-H 'Content-Type: application/json' \ +-H 'kbn-xsrf: true' \ +-H 'Authorization: ApiKey ${YOUR_API_KEY}' +-------------------------------------------------- + +[[apm-sourcemap-get-body]] +===== Response body + +[source,js] +-------------------------------------------------- +{ + "artifacts": [ + { + "type": "sourcemap", + "identifier": "foo-1.0.0", + "relative_url": "/api/fleet/artifacts/foo-1.0.0/644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456", + "body": { + "serviceName": "foo", + "serviceVersion": "1.0.0", + "bundleFilepath": "/test/e2e/general-usecase/bundle.js.map", + "sourceMap": { + "version": 3, + "file": "static/js/main.chunk.js", + "sources": [ + "fleet-source-map-client/src/index.css", + "fleet-source-map-client/src/App.js", + "webpack:///./src/index.css?bb0a", + "fleet-source-map-client/src/index.js", + "fleet-source-map-client/src/reportWebVitals.js" + ], + "sourcesContent": [ + "content" + ], + "mappings": "mapping", + "sourceRoot": "" + } + }, + "created": "2021-07-09T20:47:44.812Z", + "id": "apm:foo-1.0.0-644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456", + "compressionAlgorithm": "zlib", + "decodedSha256": "644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456", + "decodedSize": 441, + "encodedSha256": "024c72749c3e3dd411b103f7040ae62633558608f480bce4b108cf5b2275bd24", + "encodedSize": 237, + "encryptionAlgorithm": "none", + "packageName": "apm" + } + ] +} +-------------------------------------------------- + +//// +******************************************************* +//// + +[[rum-sourcemap-delete]] +==== Delete source map + +Delete a previously uploaded source map. + +[[rum-sourcemap-delete-privs]] +===== Privileges + +The user accessing this endpoint requires `All` Kibana privileges for the {beat_kib_app} feature. +For more information, see <>. + +[[apm-sourcemap-delete-req]] +===== Request + +`DELETE /api/apm/sourcemaps/:id` + +[[apm-sourcemap-delete-example]] +===== Example + +The following example deletes a source map with an id of `apm:foo-1.0.0-644fd5a9`: + +[source,curl] +-------------------------------------------------- +curl -X DELETE "http://localhost:5601/api/apm/sourcemaps/apm:foo-1.0.0-644fd5a9" \ +-H 'Content-Type: application/json' \ +-H 'kbn-xsrf: true' \ +-H 'Authorization: ApiKey ${YOUR_API_KEY}' +-------------------------------------------------- + +[[apm-sourcemap-delete-body]] +===== Response body + +[source,js] +-------------------------------------------------- +{} +-------------------------------------------------- diff --git a/docs/apm/apm-app-users.asciidoc b/docs/apm/apm-app-users.asciidoc index 9b8a9c64ac43b5..a8eb619a8eab83 100644 --- a/docs/apm/apm-app-users.asciidoc +++ b/docs/apm/apm-app-users.asciidoc @@ -3,7 +3,6 @@ == APM app users and privileges :beat_default_index_prefix: apm -:beat_kib_app: APM app :annotation_index: observability-annotations ++++ @@ -22,7 +21,7 @@ In general, there are three types of privileges you'll work with: * **Elasticsearch cluster privileges**: Manage the actions a user can perform against your cluster. * **Elasticsearch index privileges**: Control access to the data in specific indices your cluster. -* **Kibana space privileges**: Grant users write or read access to features and apps within Kibana. +* **Kibana feature privileges**: Grant users write or read access to features and apps within Kibana. Select your use-case to get started: @@ -88,18 +87,18 @@ include::./tab-widgets/apm-app-reader/widget.asciidoc[] TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]? Add the privileges under the **Data streams** tab. -. Assign space privileges to any Kibana space that the user needs access to. +. Assign feature privileges to any Kibana feature that the user needs access to. Here are two examples: + [options="header"] |==== |Type | Privilege | Purpose -| Spaces -| `Read` or `All` on the {beat_kib_app} -| Allow the use of the the {beat_kib_app} +| Kibana +| `Read` or `All` on the {beat_kib_app} feature +| Allow the use of the the {beat_kib_app} apps -| Spaces +| Kibana | `Read` or `All` on Dashboards and Discover | Allow the user to view, edit, and create dashboards, as well as browse data. |==== @@ -190,16 +189,16 @@ include::./tab-widgets/central-config-users/widget.asciidoc[] TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]? Add the privileges under the **Data streams** tab. -. Assign the `central-config-manager` role created in the previous step, and the following Kibana space privileges to -anyone who needs to manage central configurations: +. Assign the `central-config-manager` role created in the previous step, +and the following Kibana feature privileges to anyone who needs to manage central configurations: + [options="header"] |==== |Type | Privilege | Purpose -| Spaces -|`All` on {beat_kib_app} -|Allow full use of the {beat_kib_app} +| Kibana +|`All` on the {beat_kib_app} feature +|Allow full use of the {beat_kib_app} apps |==== [[apm-app-central-config-reader]] @@ -217,16 +216,16 @@ include::./tab-widgets/central-config-users/widget.asciidoc[] TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]? Add the privileges under the **Data streams** tab. -. Assign the `central-config-reader` role created in the previous step, and the following Kibana space privileges to -anyone who needs to read central configurations: +. Assign the `central-config-reader` role created in the previous step, +and the following Kibana feature privileges to anyone who needs to read central configurations: + [options="header"] |==== |Type | Privilege | Purpose -| Spaces -|`read` on the {beat_kib_app} -|Allow read access to the {beat_kib_app} +| Kibana +|`read` on the {beat_kib_app} feature +|Allow read access to the {beat_kib_app} apps |==== [[apm-app-central-config-api]] @@ -253,15 +252,15 @@ include::./tab-widgets/code.asciidoc[] Users can list, search, create, update, and delete central configurations via the APM app API. -. Assign the following Kibana space privileges: +. Assign the following Kibana feature privileges: + [options="header"] |==== |Type | Privilege | Purpose -| Spaces -|`all` on the {beat_kib_app} -|Allow all access to the {beat_kib_app} +| Kibana +|`all` on the {beat_kib_app} feature +|Allow all access to the {beat_kib_app} apps |==== [[apm-app-api-config-reader]] @@ -269,15 +268,15 @@ Users can list, search, create, update, and delete central configurations via th Sometimes a user only needs to list and search central configurations via the APM app API. -. Assign the following Kibana space privileges: +. Assign the following Kibana feature privileges: + [options="header"] |==== |Type | Privilege | Purpose -| Spaces -|`read` on the {beat_kib_app} -|Allow read access to the {beat_kib_app} +| Kibana +|`read` on the {beat_kib_app} feature +|Allow read access to the {beat_kib_app} apps |==== [[apm-app-api-annotation-manager]] @@ -310,15 +309,15 @@ and assign the following privileges: |==== . Assign the `annotation_role` created previously, -and the following Kibana space privileges to any annotation API users: +and the following Kibana feature privileges to any annotation API users: + [options="header"] |==== |Type | Privilege | Purpose -| Spaces -|`all` on the {beat_kib_app} -|Allow all access to the {beat_kib_app} +| Kibana +|`all` on the {beat_kib_app} feature +|Allow all access to the {beat_kib_app} apps |==== //LEARN MORE diff --git a/docs/apm/index.asciidoc b/docs/apm/index.asciidoc index 53ffee5e061d60..f4d35a2d554ba3 100644 --- a/docs/apm/index.asciidoc +++ b/docs/apm/index.asciidoc @@ -25,6 +25,8 @@ you can also see contextual information such as the request header, user informa system values, or custom data that you manually attached to the request. -- +:beat_kib_app: APM and User Experience + include::set-up.asciidoc[] include::getting-started.asciidoc[] From 879b1a40dc4cf54c48dd619198bb4c5587a0fab2 Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Mon, 19 Jul 2021 16:54:13 -0400 Subject: [PATCH 13/39] [Security Solution][Endpoint] Fix UI inconsistency between isolation forms and remove display of Pending isolation statuses (#106118) * comment out UI display of pending isolation statuses and associated tests * Change un-isolate form to use `EuiForm` and `EuiFormRow` * Fix: move component `displayName` to file that has that component's src --- .../endpoint_host_isolation_status.test.tsx | 3 +- .../endpoint_host_isolation_status.tsx | 180 ++++++++++-------- .../host_isolation/unisolate_form.tsx | 87 +++++---- .../context_menu_item_nav_by_rotuer.tsx | 2 + .../components/endpoint_agent_status.test.tsx | 3 +- .../view/components/table_row_actions.tsx | 2 - .../endpoint_overview/index.test.tsx | 3 +- 7 files changed, 151 insertions(+), 129 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.test.tsx b/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.test.tsx index 4ceacc40942e20..373b4d78a84cc0 100644 --- a/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.test.tsx @@ -43,7 +43,8 @@ describe('when using the EndpointHostIsolationStatus component', () => { expect(getByTestId('test').textContent).toBe('Isolated'); }); - it.each([ + // FIXME: un-skip when we bring back the pending isolation statuses + it.skip.each([ ['Isolating', { pendingIsolate: 2 }], ['Releasing', { pendingUnIsolate: 2 }], ['4 actions pending', { isIsolated: true, pendingUnIsolate: 2, pendingIsolate: 2 }], diff --git a/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.tsx b/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.tsx index 7ae7cae89f19ed..425172a5cd4604 100644 --- a/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.tsx +++ b/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/endpoint_host_isolation_status.tsx @@ -6,9 +6,9 @@ */ import React, { memo, useMemo } from 'react'; -import { EuiBadge, EuiFlexGroup, EuiFlexItem, EuiTextColor, EuiToolTip } from '@elastic/eui'; +import { EuiBadge } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import { useTestIdGenerator } from '../../../../management/components/hooks/use_test_id_generator'; +// import { useTestIdGenerator } from '../../../../management/components/hooks/use_test_id_generator'; export interface EndpointHostIsolationStatusProps { isIsolated: boolean; @@ -25,94 +25,114 @@ export interface EndpointHostIsolationStatusProps { * (`null` is returned) */ export const EndpointHostIsolationStatus = memo( - ({ isIsolated, pendingIsolate = 0, pendingUnIsolate = 0, 'data-test-subj': dataTestSubj }) => { - const getTestId = useTestIdGenerator(dataTestSubj); + ({ + isIsolated, + /* pendingIsolate = 0, pendingUnIsolate = 0,*/ 'data-test-subj': dataTestSubj, + }) => { + // const getTestId = useTestIdGenerator(dataTestSubj); return useMemo(() => { // If nothing is pending and host is not currently isolated, then render nothing - if (!isIsolated && !pendingIsolate && !pendingUnIsolate) { + if (!isIsolated) { return null; } + // if (!isIsolated && !pendingIsolate && !pendingUnIsolate) { + // return null; + // } - // If nothing is pending, but host is isolated, then show isolation badge - if (!pendingIsolate && !pendingUnIsolate) { - return ( - - - - ); - } - - // If there are multiple types of pending isolation actions, then show count of actions with tooltip that displays breakdown - if (pendingIsolate && pendingUnIsolate) { - return ( - - -

- -
- - - - - {pendingIsolate} - - - - - - {pendingUnIsolate} - - - } - > - - - - - - ); - } - - // Show 'pending [un]isolate' depending on what's pending return ( - - {pendingIsolate ? ( - - ) : ( - - )} - + ); - }, [dataTestSubj, getTestId, isIsolated, pendingIsolate, pendingUnIsolate]); + + // If nothing is pending and host is not currently isolated, then render nothing + // if (!isIsolated && !pendingIsolate && !pendingUnIsolate) { + // return null; + // } + // + // // If nothing is pending, but host is isolated, then show isolation badge + // if (!pendingIsolate && !pendingUnIsolate) { + // return ( + // + // + // + // ); + // } + // + // // If there are multiple types of pending isolation actions, then show count of actions with tooltip that displays breakdown + // if (pendingIsolate && pendingUnIsolate) { + // return ( + // + // + //
+ // + //
+ // + // + // + // + // {pendingIsolate} + // + // + // + // + // + // {pendingUnIsolate} + // + // + // } + // > + // + // + // + // + // + // ); + // } + // + // // Show 'pending [un]isolate' depending on what's pending + // return ( + // + // + // {pendingIsolate ? ( + // + // ) : ( + // + // )} + // + // + // ); + }, [dataTestSubj, isIsolated /* , getTestId , pendingIsolate, pendingUnIsolate*/]); } ); diff --git a/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/unisolate_form.tsx b/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/unisolate_form.tsx index 98006524844c4d..ac650edca43e49 100644 --- a/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/unisolate_form.tsx +++ b/x-pack/plugins/security_solution/public/common/components/endpoint/host_isolation/unisolate_form.tsx @@ -11,10 +11,10 @@ import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, - EuiSpacer, + EuiForm, + EuiFormRow, EuiText, EuiTextArea, - EuiTitle, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { CANCEL, COMMENT, COMMENT_PLACEHOLDER, CONFIRM, UNISOLATE, ISOLATED } from './translations'; @@ -30,50 +30,49 @@ export const EndpointUnisolateForm = memo( ); return ( - <> - -

- {hostName}, - isolated: {ISOLATED}, - unisolate: {UNISOLATE}, - }} - />{' '} - {messageAppend} -

- + + + +

+ {hostName}, + isolated: {ISOLATED}, + unisolate: {UNISOLATE}, + }} + />{' '} + {messageAppend} +

+ + - + + + - -

{COMMENT}

- - - - - - - - - {CANCEL} - - - - - {CONFIRM} - - - - + + + + + {CANCEL} + + + + + {CONFIRM} + + + + + ); } ); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/context_menu_item_nav_by_rotuer.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/context_menu_item_nav_by_rotuer.tsx index ac1b83bdc493bc..f4f9a7542415df 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/context_menu_item_nav_by_rotuer.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/context_menu_item_nav_by_rotuer.tsx @@ -34,3 +34,5 @@ export const ContextMenuItemNavByRouter = memo( ); } ); + +ContextMenuItemNavByRouter.displayName = 'EuiContextMenuItemNavByRouter'; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/endpoint_agent_status.test.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/endpoint_agent_status.test.tsx index 0b5ff7cc4da0f8..be7fef156815b3 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/endpoint_agent_status.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/endpoint_agent_status.test.tsx @@ -57,7 +57,8 @@ describe('When using the EndpointAgentStatus component', () => { expect(renderResult.getByTestId('rowHostStatus').textContent).toEqual(expectedLabel); }); - describe('and host is isolated or pending isolation', () => { + // FIXME: un-skip test once Islation pending statuses are supported + describe.skip('and host is isolated or pending isolation', () => { beforeEach(async () => { // Ensure pending action api sets pending action for the test endpoint metadata const pendingActionsResponseProvider = httpMocks.responseProvider.pendingActions.getMockImplementation(); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/table_row_actions.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/table_row_actions.tsx index 94303c43cd4da3..5a2ad6cf4c60b1 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/table_row_actions.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/table_row_actions.tsx @@ -62,5 +62,3 @@ export const TableRowActions = memo(({ endpointMetadata }) ); }); TableRowActions.displayName = 'EndpointTableRowActions'; - -ContextMenuItemNavByRouter.displayName = 'EuiContextMenuItemNavByRouter'; diff --git a/x-pack/plugins/security_solution/public/overview/components/host_overview/endpoint_overview/index.test.tsx b/x-pack/plugins/security_solution/public/overview/components/host_overview/endpoint_overview/index.test.tsx index 6a0e7c381664c0..7f447670fd1e14 100644 --- a/x-pack/plugins/security_solution/public/overview/components/host_overview/endpoint_overview/index.test.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/host_overview/endpoint_overview/index.test.tsx @@ -71,7 +71,8 @@ describe('EndpointOverview Component', () => { expect(findData.at(3).text()).toEqual('HealthyIsolated'); }); - test.each([ + // FIXME: un-skip once pending isolation status are supported again + test.skip.each([ ['isolate', 'Isolating'], ['unisolate', 'Releasing'], ])('it shows pending %s status', (action, expectedLabel) => { From 591c11988d0db23f3fb4ef9266cc2f7ae27a8388 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 20 Jul 2021 00:09:05 +0100 Subject: [PATCH 14/39] skip flaky suite (#106121) --- test/functional/apps/visualize/_metric_chart.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/functional/apps/visualize/_metric_chart.ts b/test/functional/apps/visualize/_metric_chart.ts index 7853a3a845bfc4..6f6767619f486a 100644 --- a/test/functional/apps/visualize/_metric_chart.ts +++ b/test/functional/apps/visualize/_metric_chart.ts @@ -17,7 +17,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const inspector = getService('inspector'); const PageObjects = getPageObjects(['visualize', 'visEditor', 'visChart', 'timePicker']); - describe('metric chart', function () { + // FLAKY: https://github.com/elastic/kibana/issues/106121 + describe.skip('metric chart', function () { before(async function () { await PageObjects.visualize.initTests(); log.debug('navigateToApp visualize'); From a34a15ea3d380282132ce2d2f2914e7165ab50e8 Mon Sep 17 00:00:00 2001 From: Davis Plumlee <56367316+dplumlee@users.noreply.github.com> Date: Mon, 19 Jul 2021 22:37:42 -0600 Subject: [PATCH 15/39] [Security Solution] Invalid kql query timeline refresh bug (#105525) * poc test * adds disable for refresh button Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../components/super_date_picker/index.tsx | 3 ++ .../timeline/query_tab_content/index.tsx | 36 +++++++++++++------ .../components/timeline/refetch_timeline.tsx | 8 +++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.tsx b/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.tsx index c57ae1d11b8daf..04e4203df1a99b 100644 --- a/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.tsx @@ -91,6 +91,7 @@ export const SuperDatePickerComponent = React.memo( timelineId, toStr, updateReduxTime, + disabled, }) => { const [recentlyUsedRanges, setRecentlyUsedRanges] = useState( [] @@ -201,6 +202,7 @@ export const SuperDatePickerComponent = React.memo( refreshInterval={duration} showUpdateButton={true} start={startDate} + isDisabled={disabled} /> ); }, @@ -216,6 +218,7 @@ export const SuperDatePickerComponent = React.memo( prevProps.startAutoReload === nextProps.startAutoReload && prevProps.stopAutoReload === nextProps.stopAutoReload && prevProps.timelineId === nextProps.timelineId && + prevProps.disabled === nextProps.disabled && prevProps.toStr === nextProps.toStr && prevProps.updateReduxTime === nextProps.updateReduxTime && deepEqual(prevProps.kqlQuery, nextProps.kqlQuery) && diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx index c2e47edeae2029..abbb991c274da1 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/index.tsx @@ -220,14 +220,21 @@ export const QueryTabContentComponent: React.FC = ({ }); const isBlankTimeline: boolean = - isEmpty(dataProviders) && isEmpty(filters) && isEmpty(kqlQuery.query); - - const canQueryTimeline = () => - combinedQueries != null && - loadingSourcerer != null && - !loadingSourcerer && - !isEmpty(start) && - !isEmpty(end); + isEmpty(dataProviders) && + isEmpty(filters) && + isEmpty(kqlQuery.query) && + combinedQueries?.filterQuery === undefined; + + const canQueryTimeline = useMemo( + () => + combinedQueries != null && + loadingSourcerer != null && + !loadingSourcerer && + !isEmpty(start) && + !isEmpty(end) && + combinedQueries?.filterQuery !== undefined, + [combinedQueries, end, loadingSourcerer, start] + ); const getTimelineQueryFields = () => { const columnsHeader = isEmpty(columns) ? defaultHeaders : columns; @@ -264,7 +271,7 @@ export const QueryTabContentComponent: React.FC = ({ limit: itemsPerPage, filterQuery: combinedQueries?.filterQuery, startDate: start, - skip: !canQueryTimeline() || combinedQueries?.filterQuery === undefined, + skip: !canQueryTimeline, sort: timelineQuerySortField, timerangeKind, }); @@ -290,6 +297,10 @@ export const QueryTabContentComponent: React.FC = ({ ); }, [loadingSourcerer, timelineId, isQueryLoading, dispatch]); + const isDatePickerDisabled = useMemo(() => { + return (combinedQueries && combinedQueries.kqlError != null) || false; + }, [combinedQueries]); + const leadingControlColumns: ControlColumnProps[] = [defaultControlColumn]; const trailingControlColumns: ControlColumnProps[] = []; @@ -304,6 +315,7 @@ export const QueryTabContentComponent: React.FC = ({ inspect={inspect} loading={isQueryLoading} refetch={refetch} + skip={!canQueryTimeline} /> @@ -323,7 +335,11 @@ export const QueryTabContentComponent: React.FC = ({ /> )} - + diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/refetch_timeline.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/refetch_timeline.tsx index 108013c9065220..0b51f0183bbb46 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/refetch_timeline.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/refetch_timeline.tsx @@ -18,6 +18,7 @@ export interface TimelineRefetchProps { inspect: inputsModel.InspectQuery | null; loading: boolean; refetch: inputsModel.Refetch; + skip?: boolean; } const TimelineRefetchComponent: React.FC = ({ @@ -26,12 +27,15 @@ const TimelineRefetchComponent: React.FC = ({ inspect, loading, refetch, + skip, }) => { const dispatch = useDispatch(); useEffect(() => { - dispatch(inputsActions.setQuery({ id, inputId, inspect, loading, refetch })); - }, [dispatch, id, inputId, loading, refetch, inspect]); + if (!skip) { + dispatch(inputsActions.setQuery({ id, inputId, inspect, loading, refetch })); + } + }, [dispatch, id, inputId, loading, refetch, inspect, skip]); return null; }; From 237256a4bad936b74c56dd9f718b57526109fa58 Mon Sep 17 00:00:00 2001 From: Aleh Zasypkin Date: Tue, 20 Jul 2021 06:52:23 +0200 Subject: [PATCH 16/39] Introduce `preboot` lifecycle stage (#103636) --- .github/CODEOWNERS | 10 +- ...n-core-server.corepreboot.elasticsearch.md | 13 + ...ana-plugin-core-server.corepreboot.http.md | 13 + .../kibana-plugin-core-server.corepreboot.md | 22 + ...-plugin-core-server.corepreboot.preboot.md | 13 + .../kibana-plugin-core-server.coresetup.md | 2 +- ...ana-plugin-core-server.discoveredplugin.md | 1 + ...lugin-core-server.discoveredplugin.type.md | 13 + ...earchconfigpreboot.credentialsspecified.md | 13 + ...server.elasticsearchconfigpreboot.hosts.md | 13 + ...-core-server.elasticsearchconfigpreboot.md | 21 + ...rver.elasticsearchservicepreboot.config.md | 22 + ...lasticsearchservicepreboot.createclient.md | 23 + ...core-server.elasticsearchservicepreboot.md | 20 + ...core-server.httpservicepreboot.basepath.md | 13 + ...a-plugin-core-server.httpservicepreboot.md | 82 ++ ...erver.httpservicepreboot.registerroutes.md | 40 + .../core/server/kibana-plugin-core-server.md | 11 +- .../kibana-plugin-core-server.plugin.md | 2 +- ...na-plugin-core-server.plugininitializer.md | 2 +- ...ore-server.plugininitializercontext.env.md | 1 + ...in-core-server.plugininitializercontext.md | 2 +- ...ibana-plugin-core-server.pluginmanifest.md | 1 + ...-plugin-core-server.pluginmanifest.type.md | 13 + .../kibana-plugin-core-server.plugintype.md | 20 + ...kibana-plugin-core-server.prebootplugin.md | 21 + ...-plugin-core-server.prebootplugin.setup.md | 23 + ...a-plugin-core-server.prebootplugin.stop.md | 15 + ...otservicepreboot.holdsetupuntilresolved.md | 15 + ...ver.prebootservicepreboot.issetuponhold.md | 13 + ...lugin-core-server.prebootservicepreboot.md | 45 + examples/preboot_example/README.md | 3 + examples/preboot_example/kibana.json | 16 + examples/preboot_example/public/app.tsx | 218 ++++ examples/preboot_example/public/config.ts | 11 + examples/preboot_example/public/index.ts | 13 + examples/preboot_example/public/plugin.tsx | 35 + examples/preboot_example/server/config.ts | 18 + examples/preboot_example/server/index.ts | 20 + examples/preboot_example/server/plugin.ts | 135 ++ examples/preboot_example/tsconfig.json | 12 + .../kbn-config/src/config_service.test.ts | 11 + packages/kbn-config/src/config_service.ts | 15 +- packages/kbn-config/src/index.ts | 2 +- src/core/TESTING.md | 2 + src/core/public/plugins/plugin.test.ts | 3 +- .../public/plugins/plugins_service.test.ts | 3 +- src/core/server/bootstrap.ts | 27 +- .../capabilities/capabilities_service.mock.ts | 1 + .../capabilities/capabilities_service.test.ts | 31 +- .../capabilities/capabilities_service.ts | 18 +- .../capabilities_service.test.ts | 5 +- src/core/server/capabilities/routes/index.ts | 5 +- .../config/ensure_valid_configuration.test.ts | 14 + .../config/ensure_valid_configuration.ts | 9 +- .../config_deprecation.test.ts | 2 + .../server/context/context_service.mock.ts | 12 +- .../server/context/context_service.test.ts | 13 + src/core/server/context/context_service.ts | 15 + src/core/server/context/index.ts | 2 +- src/core/server/core_app/core_app.test.ts | 73 +- src/core/server/core_app/core_app.ts | 138 ++- .../integration_tests/bundle_routes.test.ts | 3 +- .../integration_tests/core_app_routes.test.ts | 1 + .../default_route_provider_config.test.ts | 1 + .../integration_tests/static_assets.test.ts | 1 + .../elasticsearch_service.mock.ts | 27 +- .../elasticsearch_service.test.ts | 112 +- .../elasticsearch/elasticsearch_service.ts | 44 +- src/core/server/elasticsearch/index.ts | 3 + src/core/server/elasticsearch/types.ts | 58 + .../environment/environment_service.mock.ts | 22 +- .../environment/environment_service.test.ts | 23 +- .../server/environment/environment_service.ts | 17 +- src/core/server/environment/index.ts | 5 +- .../integration_tests/tracing.test.ts | 2 + .../__snapshots__/http_service.test.ts.snap | 2 +- src/core/server/http/http_service.mock.ts | 36 + src/core/server/http/http_service.test.ts | 224 +++- src/core/server/http/http_service.ts | 162 ++- src/core/server/http/index.ts | 2 + .../integration_tests/core_services.test.ts | 3 + .../http/integration_tests/http_auth.test.ts | 1 + .../http/integration_tests/lifecycle.test.ts | 3 +- .../lifecycle_handlers.test.ts | 1 + .../http/integration_tests/logging.test.ts | 13 + .../http/integration_tests/preboot.test.ts | 146 +++ .../http/integration_tests/request.test.ts | 3 +- .../http/integration_tests/router.test.ts | 3 +- src/core/server/http/types.ts | 110 +- .../http_resources_service.mock.ts | 7 +- .../http_resources_service.test.ts | 361 +++--- .../http_resources/http_resources_service.ts | 19 +- src/core/server/http_resources/index.ts | 1 + .../http_resources_service.test.ts | 1 + src/core/server/http_resources/types.ts | 8 +- src/core/server/i18n/i18n_service.mock.ts | 1 + src/core/server/i18n/i18n_service.test.ts | 63 +- src/core/server/i18n/i18n_service.ts | 32 +- src/core/server/index.ts | 29 +- src/core/server/internal_types.ts | 31 +- .../legacy/integration_tests/logging.test.ts | 4 + src/core/server/logging/index.ts | 6 +- .../logging/integration_tests/logging.test.ts | 2 + .../rolling_file_appender.test.ts | 3 + .../server/logging/logging_service.mock.ts | 8 + .../server/logging/logging_service.test.ts | 133 +- src/core/server/logging/logging_service.ts | 20 +- .../server_collector.test.ts | 1 + src/core/server/mocks.ts | 39 +- .../discovery/plugin_manifest_parser.test.ts | 28 + .../discovery/plugin_manifest_parser.ts | 14 +- .../discovery/plugins_discovery.test.ts | 49 +- src/core/server/plugins/index.ts | 7 +- .../integration_tests/plugins_service.test.ts | 12 +- src/core/server/plugins/plugin.test.ts | 30 +- src/core/server/plugins/plugin.ts | 30 +- .../server/plugins/plugin_context.test.ts | 75 +- src/core/server/plugins/plugin_context.ts | 45 +- .../server/plugins/plugins_service.mock.ts | 1 + .../server/plugins/plugins_service.test.ts | 1102 ++++++++++++----- src/core/server/plugins/plugins_service.ts | 155 ++- .../plugins/plugins_system.test.mocks.ts | 2 + .../server/plugins/plugins_system.test.ts | 132 +- src/core/server/plugins/plugins_system.ts | 56 +- src/core/server/plugins/types.ts | 39 +- src/core/server/preboot/index.ts | 10 + .../server/preboot/preboot_service.mock.ts | 49 + .../server/preboot/preboot_service.test.ts | 191 +++ src/core/server/preboot/preboot_service.ts | 58 + src/core/server/preboot/types.ts | 85 ++ ...preboot_core_route_handler_context.test.ts | 36 + .../preboot_core_route_handler_context.ts | 25 + src/core/server/rendering/__mocks__/params.ts | 12 +- .../rendering/__mocks__/rendering_service.ts | 7 +- .../rendering_service.test.ts.snap | 259 ++++ .../bootstrap/get_plugin_bundle_paths.test.ts | 3 +- .../rendering/rendering_service.mock.ts | 10 +- .../rendering/rendering_service.test.ts | 202 +-- .../server/rendering/rendering_service.tsx | 168 +-- src/core/server/rendering/types.ts | 16 +- src/core/server/root/index.test.mocks.ts | 1 + src/core/server/root/index.test.ts | 63 +- src/core/server/root/index.ts | 12 +- .../actions/integration_tests/actions.test.ts | 1 + .../integration_tests/es_errors.test.ts | 1 + .../integration_tests/cleanup.test.ts | 1 + .../corrupt_outdated_docs.test.ts | 1 + .../integration_tests/migration.test.ts | 3 +- .../migration_7.7.2_xpack_100k.test.ts | 3 +- ...igration_7_13_0_transform_failures.test.ts | 1 + .../migration_7_13_0_unknown_types.test.ts | 3 + .../multiple_es_nodes.test.ts | 1 + .../multiple_kibana_nodes.test.ts | 14 +- .../integration_tests/outdated_docs.test.ts | 1 + .../integration_tests/rewriting_id.test.ts | 1 + .../type_migration_failure.test.ts | 1 + .../type_registrations.test.ts | 1 + .../routes/integration_tests/get.test.ts | 3 +- .../routes/integration_tests/migrate.test.ts | 1 + .../routes/integration_tests/resolve.test.ts | 3 +- .../server/saved_objects/routes/test_utils.ts | 3 +- .../lib/integration_tests/repository.test.ts | 1 + src/core/server/server.api.md | 63 +- src/core/server/server.test.mocks.ts | 6 + src/core/server/server.test.ts | 88 +- src/core/server/server.ts | 84 +- .../routes/integration_tests/status.test.ts | 2 + src/core/server/status/status_service.test.ts | 41 + src/core/server/status/status_service.ts | 6 +- .../ui_settings/base_ui_settings_client.ts | 93 ++ src/core/server/ui_settings/index.ts | 1 + .../integration_tests/routes.test.ts | 1 + src/core/server/ui_settings/types.ts | 8 + .../server/ui_settings/ui_settings_client.ts | 65 +- .../ui_settings_defaults_client.test.ts | 188 +++ .../ui_settings_defaults_client.ts | 58 + .../ui_settings/ui_settings_service.mock.ts | 14 + .../ui_settings_service.test.mock.ts | 5 + .../ui_settings/ui_settings_service.test.ts | 34 +- .../server/ui_settings/ui_settings_service.ts | 25 +- src/core/test_helpers/kbn_server.ts | 1 + .../kibana_usage_collection/kibana.json | 3 - .../integration_tests/daily_rollups.test.ts | 1 + .../routes/integration_tests/stats.test.ts | 1 + .../create_apm_event_client/index.test.ts | 3 + .../server/integration_tests/router.test.ts | 5 + .../on_post_auth_interceptor.test.ts | 1 + .../on_request_interceptor.test.ts | 1 + .../server/routes/settings.test.ts | 1 + .../saml/saml_provider/server/index.ts | 4 +- .../oidc/oidc_provider/server/index.ts | 4 +- .../saml/saml_provider/server/index.ts | 4 +- .../common/test_endpoints/server/index.ts | 4 +- 194 files changed, 5848 insertions(+), 1192 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.corepreboot.elasticsearch.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.corepreboot.http.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.corepreboot.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.corepreboot.preboot.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.discoveredplugin.type.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.credentialsspecified.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.hosts.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.config.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.createclient.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.basepath.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.registerroutes.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.pluginmanifest.type.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.plugintype.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.prebootplugin.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.prebootplugin.setup.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.prebootplugin.stop.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.holdsetupuntilresolved.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.issetuponhold.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.md create mode 100644 examples/preboot_example/README.md create mode 100644 examples/preboot_example/kibana.json create mode 100644 examples/preboot_example/public/app.tsx create mode 100644 examples/preboot_example/public/config.ts create mode 100644 examples/preboot_example/public/index.ts create mode 100644 examples/preboot_example/public/plugin.tsx create mode 100644 examples/preboot_example/server/config.ts create mode 100644 examples/preboot_example/server/index.ts create mode 100644 examples/preboot_example/server/plugin.ts create mode 100644 examples/preboot_example/tsconfig.json create mode 100644 src/core/server/http/integration_tests/preboot.test.ts create mode 100644 src/core/server/preboot/index.ts create mode 100644 src/core/server/preboot/preboot_service.mock.ts create mode 100644 src/core/server/preboot/preboot_service.test.ts create mode 100644 src/core/server/preboot/preboot_service.ts create mode 100644 src/core/server/preboot/types.ts create mode 100644 src/core/server/preboot_core_route_handler_context.test.ts create mode 100644 src/core/server/preboot_core_route_handler_context.ts create mode 100644 src/core/server/ui_settings/base_ui_settings_client.ts create mode 100644 src/core/server/ui_settings/ui_settings_defaults_client.test.ts create mode 100644 src/core/server/ui_settings/ui_settings_defaults_client.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 168db2c4efb84c..0744112650c23f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -249,14 +249,14 @@ #CC# /x-pack/plugins/translations/ @elastic/kibana-localization @elastic/kibana-core # Security -/src/core/server/csp/ @elastic/kibana-security @elastic/kibana-core +/src/core/server/csp/ @elastic/kibana-security @elastic/kibana-core /src/plugins/security_oss/ @elastic/kibana-security /src/plugins/spaces_oss/ @elastic/kibana-security /src/plugins/user_setup/ @elastic/kibana-security /test/security_functional/ @elastic/kibana-security -/x-pack/plugins/spaces/ @elastic/kibana-security -/x-pack/plugins/encrypted_saved_objects/ @elastic/kibana-security -/x-pack/plugins/security/ @elastic/kibana-security +/x-pack/plugins/spaces/ @elastic/kibana-security +/x-pack/plugins/encrypted_saved_objects/ @elastic/kibana-security +/x-pack/plugins/security/ @elastic/kibana-security /x-pack/test/api_integration/apis/security/ @elastic/kibana-security /x-pack/test/ui_capabilities/ @elastic/kibana-security /x-pack/test/encrypted_saved_objects_api_integration/ @elastic/kibana-security @@ -265,6 +265,8 @@ /x-pack/test/security_functional/ @elastic/kibana-security /x-pack/test/spaces_api_integration/ @elastic/kibana-security /x-pack/test/saved_object_api_integration/ @elastic/kibana-security +/src/core/server/csp/ @elastic/kibana-security @elastic/kibana-core +/examples/preboot_example/ @elastic/kibana-security @elastic/kibana-core #CC# /x-pack/plugins/security/ @elastic/kibana-security # Kibana Alerting Services diff --git a/docs/development/core/server/kibana-plugin-core-server.corepreboot.elasticsearch.md b/docs/development/core/server/kibana-plugin-core-server.corepreboot.elasticsearch.md new file mode 100644 index 00000000000000..7d3b5296b5988f --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.corepreboot.elasticsearch.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [CorePreboot](./kibana-plugin-core-server.corepreboot.md) > [elasticsearch](./kibana-plugin-core-server.corepreboot.elasticsearch.md) + +## CorePreboot.elasticsearch property + +[ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) + +Signature: + +```typescript +elasticsearch: ElasticsearchServicePreboot; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.corepreboot.http.md b/docs/development/core/server/kibana-plugin-core-server.corepreboot.http.md new file mode 100644 index 00000000000000..0df643c6f133b4 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.corepreboot.http.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [CorePreboot](./kibana-plugin-core-server.corepreboot.md) > [http](./kibana-plugin-core-server.corepreboot.http.md) + +## CorePreboot.http property + +[HttpServicePreboot](./kibana-plugin-core-server.httpservicepreboot.md) + +Signature: + +```typescript +http: HttpServicePreboot; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.corepreboot.md b/docs/development/core/server/kibana-plugin-core-server.corepreboot.md new file mode 100644 index 00000000000000..475b5f109d58c3 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.corepreboot.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [CorePreboot](./kibana-plugin-core-server.corepreboot.md) + +## CorePreboot interface + +Context passed to the `setup` method of `preboot` plugins. + +Signature: + +```typescript +export interface CorePreboot +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [elasticsearch](./kibana-plugin-core-server.corepreboot.elasticsearch.md) | ElasticsearchServicePreboot | [ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) | +| [http](./kibana-plugin-core-server.corepreboot.http.md) | HttpServicePreboot | [HttpServicePreboot](./kibana-plugin-core-server.httpservicepreboot.md) | +| [preboot](./kibana-plugin-core-server.corepreboot.preboot.md) | PrebootServicePreboot | [PrebootServicePreboot](./kibana-plugin-core-server.prebootservicepreboot.md) | + diff --git a/docs/development/core/server/kibana-plugin-core-server.corepreboot.preboot.md b/docs/development/core/server/kibana-plugin-core-server.corepreboot.preboot.md new file mode 100644 index 00000000000000..3780a92053a5e5 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.corepreboot.preboot.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [CorePreboot](./kibana-plugin-core-server.corepreboot.md) > [preboot](./kibana-plugin-core-server.corepreboot.preboot.md) + +## CorePreboot.preboot property + +[PrebootServicePreboot](./kibana-plugin-core-server.prebootservicepreboot.md) + +Signature: + +```typescript +preboot: PrebootServicePreboot; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.coresetup.md b/docs/development/core/server/kibana-plugin-core-server.coresetup.md index a66db46adf0f73..b03101b4d9fe64 100644 --- a/docs/development/core/server/kibana-plugin-core-server.coresetup.md +++ b/docs/development/core/server/kibana-plugin-core-server.coresetup.md @@ -4,7 +4,7 @@ ## CoreSetup interface -Context passed to the plugins `setup` method. +Context passed to the `setup` method of `standard` plugins. Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md index b88a179c5c4b3a..042f2d1485618e 100644 --- a/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md +++ b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md @@ -21,4 +21,5 @@ export interface DiscoveredPlugin | [optionalPlugins](./kibana-plugin-core-server.discoveredplugin.optionalplugins.md) | readonly PluginName[] | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. | | [requiredBundles](./kibana-plugin-core-server.discoveredplugin.requiredbundles.md) | readonly PluginName[] | List of plugin ids that this plugin's UI code imports modules from that are not in requiredPlugins. | | [requiredPlugins](./kibana-plugin-core-server.discoveredplugin.requiredplugins.md) | readonly PluginName[] | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. | +| [type](./kibana-plugin-core-server.discoveredplugin.type.md) | PluginType | Type of the plugin, defaults to standard. | diff --git a/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.type.md b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.type.md new file mode 100644 index 00000000000000..0a33be0d63f5c0 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.type.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [DiscoveredPlugin](./kibana-plugin-core-server.discoveredplugin.md) > [type](./kibana-plugin-core-server.discoveredplugin.type.md) + +## DiscoveredPlugin.type property + +Type of the plugin, defaults to `standard`. + +Signature: + +```typescript +readonly type: PluginType; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.credentialsspecified.md b/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.credentialsspecified.md new file mode 100644 index 00000000000000..df99d5ec4b831a --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.credentialsspecified.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ElasticsearchConfigPreboot](./kibana-plugin-core-server.elasticsearchconfigpreboot.md) > [credentialsSpecified](./kibana-plugin-core-server.elasticsearchconfigpreboot.credentialsspecified.md) + +## ElasticsearchConfigPreboot.credentialsSpecified property + +Indicates whether Elasticsearch configuration includes credentials (`username`, `password` or `serviceAccountToken`). + +Signature: + +```typescript +readonly credentialsSpecified: boolean; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.hosts.md b/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.hosts.md new file mode 100644 index 00000000000000..e9ad47b61419e7 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.hosts.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ElasticsearchConfigPreboot](./kibana-plugin-core-server.elasticsearchconfigpreboot.md) > [hosts](./kibana-plugin-core-server.elasticsearchconfigpreboot.hosts.md) + +## ElasticsearchConfigPreboot.hosts property + +Hosts that the client will connect to. If sniffing is enabled, this list will be used as seeds to discover the rest of your cluster. + +Signature: + +```typescript +readonly hosts: string[]; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.md b/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.md new file mode 100644 index 00000000000000..bbccea80b672fe --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.elasticsearchconfigpreboot.md @@ -0,0 +1,21 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ElasticsearchConfigPreboot](./kibana-plugin-core-server.elasticsearchconfigpreboot.md) + +## ElasticsearchConfigPreboot interface + +A limited set of Elasticsearch configuration entries exposed to the `preboot` plugins at `setup`. + +Signature: + +```typescript +export interface ElasticsearchConfigPreboot +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [credentialsSpecified](./kibana-plugin-core-server.elasticsearchconfigpreboot.credentialsspecified.md) | boolean | Indicates whether Elasticsearch configuration includes credentials (username, password or serviceAccountToken). | +| [hosts](./kibana-plugin-core-server.elasticsearchconfigpreboot.hosts.md) | string[] | Hosts that the client will connect to. If sniffing is enabled, this list will be used as seeds to discover the rest of your cluster. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.config.md b/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.config.md new file mode 100644 index 00000000000000..12a32b4544abaf --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.config.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) > [config](./kibana-plugin-core-server.elasticsearchservicepreboot.config.md) + +## ElasticsearchServicePreboot.config property + +A limited set of Elasticsearch configuration entries. + +Signature: + +```typescript +readonly config: Readonly; +``` + +## Example + + +```js +const { hosts, credentialsSpecified } = core.elasticsearch.config; + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.createclient.md b/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.createclient.md new file mode 100644 index 00000000000000..d14e3e4efa400a --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.createclient.md @@ -0,0 +1,23 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) > [createClient](./kibana-plugin-core-server.elasticsearchservicepreboot.createclient.md) + +## ElasticsearchServicePreboot.createClient property + +Create application specific Elasticsearch cluster API client with customized config. See [IClusterClient](./kibana-plugin-core-server.iclusterclient.md). + +Signature: + +```typescript +readonly createClient: (type: string, clientConfig?: Partial) => ICustomClusterClient; +``` + +## Example + + +```js +const client = elasticsearch.createClient('my-app-name', config); +const data = await client.asInternalUser.search(); + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.md b/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.md new file mode 100644 index 00000000000000..bf458004b488bd --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.elasticsearchservicepreboot.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) + +## ElasticsearchServicePreboot interface + + +Signature: + +```typescript +export interface ElasticsearchServicePreboot +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [config](./kibana-plugin-core-server.elasticsearchservicepreboot.config.md) | Readonly<ElasticsearchConfigPreboot> | A limited set of Elasticsearch configuration entries. | +| [createClient](./kibana-plugin-core-server.elasticsearchservicepreboot.createclient.md) | (type: string, clientConfig?: Partial<ElasticsearchClientConfig>) => ICustomClusterClient | Create application specific Elasticsearch cluster API client with customized config. See [IClusterClient](./kibana-plugin-core-server.iclusterclient.md). | + diff --git a/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.basepath.md b/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.basepath.md new file mode 100644 index 00000000000000..9864f67d70a43d --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.basepath.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [HttpServicePreboot](./kibana-plugin-core-server.httpservicepreboot.md) > [basePath](./kibana-plugin-core-server.httpservicepreboot.basepath.md) + +## HttpServicePreboot.basePath property + +Access or manipulate the Kibana base path See [IBasePath](./kibana-plugin-core-server.ibasepath.md). + +Signature: + +```typescript +basePath: IBasePath; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.md b/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.md new file mode 100644 index 00000000000000..b4adf454a480f2 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.md @@ -0,0 +1,82 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [HttpServicePreboot](./kibana-plugin-core-server.httpservicepreboot.md) + +## HttpServicePreboot interface + +Kibana HTTP Service provides an abstraction to work with the HTTP stack at the `preboot` stage. This functionality allows Kibana to serve user requests even before Kibana becomes fully operational. Only Core and `preboot` plugins can define HTTP routes at this stage. + +Signature: + +```typescript +export interface HttpServicePreboot +``` + +## Example + +To handle an incoming request in your preboot plugin you should: - Use `@kbn/config-schema` package to create a schema to validate the request `params`, `query`, and `body`. Every incoming request will be validated against the created schema. If validation failed, the request is rejected with `400` status and `Bad request` error without calling the route's handler. To opt out of validating the request, specify `false`. + +```ts +import { schema, TypeOf } from '@kbn/config-schema'; +const validate = { + params: schema.object({ + id: schema.string(), + }), +}; + +``` +- Declare a function to respond to incoming request. The function will receive `request` object containing request details: url, headers, matched route, as well as validated `params`, `query`, `body`. And `response` object instructing HTTP server to create HTTP response with information sent back to the client as the response body, headers, and HTTP status. Any exception raised during the handler call will generate `500 Server error` response and log error details for further investigation. See below for returning custom error responses. + +```ts +const handler = async (context: RequestHandlerContext, request: KibanaRequest, response: ResponseFactory) => { + const data = await findObject(request.params.id); + // creates a command to respond with 'not found' error + if (!data) { + return response.notFound(); + } + // creates a command to send found data to the client and set response headers + return response.ok({ + body: data, + headers: { 'content-type': 'application/json' } + }); +} + +``` +\* - Acquire `preboot` [IRouter](./kibana-plugin-core-server.irouter.md) instance and register route handler for GET request to 'path/{id}' path. + +```ts +import { schema, TypeOf } from '@kbn/config-schema'; + +const validate = { + params: schema.object({ + id: schema.string(), + }), +}; + +httpPreboot.registerRoutes('my-plugin', (router) => { + router.get({ path: 'path/{id}', validate }, async (context, request, response) => { + const data = await findObject(request.params.id); + if (!data) { + return response.notFound(); + } + return response.ok({ + body: data, + headers: { 'content-type': 'application/json' } + }); + }); +}); + +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [basePath](./kibana-plugin-core-server.httpservicepreboot.basepath.md) | IBasePath | Access or manipulate the Kibana base path See [IBasePath](./kibana-plugin-core-server.ibasepath.md). | + +## Methods + +| Method | Description | +| --- | --- | +| [registerRoutes(path, callback)](./kibana-plugin-core-server.httpservicepreboot.registerroutes.md) | Provides ability to acquire preboot [IRouter](./kibana-plugin-core-server.irouter.md) instance for a particular top-level path and register handler functions for any number of nested routes. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.registerroutes.md b/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.registerroutes.md new file mode 100644 index 00000000000000..c188f0ba0ce944 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.httpservicepreboot.registerroutes.md @@ -0,0 +1,40 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [HttpServicePreboot](./kibana-plugin-core-server.httpservicepreboot.md) > [registerRoutes](./kibana-plugin-core-server.httpservicepreboot.registerroutes.md) + +## HttpServicePreboot.registerRoutes() method + +Provides ability to acquire `preboot` [IRouter](./kibana-plugin-core-server.irouter.md) instance for a particular top-level path and register handler functions for any number of nested routes. + +Signature: + +```typescript +registerRoutes(path: string, callback: (router: IRouter) => void): void; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| path | string | | +| callback | (router: IRouter) => void | | + +Returns: + +`void` + +## Remarks + +Each route can have only one handler function, which is executed when the route is matched. See the [IRouter](./kibana-plugin-core-server.irouter.md) documentation for more information. + +## Example + + +```ts +registerRoutes('my-plugin', (router) => { + // handler is called when '/my-plugin/path' resource is requested with `GET` method + router.get({ path: '/path', validate: false }, (context, req, res) => res.ok({ content: 'ok' })); +}); + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 4a203f10e7cd36..a26f8bd7b15944 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -41,6 +41,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | --- | --- | | [AuthResultType](./kibana-plugin-core-server.authresulttype.md) | | | [AuthStatus](./kibana-plugin-core-server.authstatus.md) | Status indicating an outcome of the authentication. | +| [PluginType](./kibana-plugin-core-server.plugintype.md) | | ## Interfaces @@ -60,7 +61,8 @@ The plugin integrates with the core system via lifecycle events: `setup` | [CapabilitiesSetup](./kibana-plugin-core-server.capabilitiessetup.md) | APIs to manage the [Capabilities](./kibana-plugin-core-server.capabilities.md) that will be used by the application.Plugins relying on capabilities to toggle some of their features should register them during the setup phase using the registerProvider method.Plugins having the responsibility to restrict capabilities depending on a given context should register their capabilities switcher using the registerSwitcher method.Refers to the methods documentation for complete description and examples. | | [CapabilitiesStart](./kibana-plugin-core-server.capabilitiesstart.md) | APIs to access the application [Capabilities](./kibana-plugin-core-server.capabilities.md). | | [ContextSetup](./kibana-plugin-core-server.contextsetup.md) | An object that handles registration of context providers and configuring handlers with context. | -| [CoreSetup](./kibana-plugin-core-server.coresetup.md) | Context passed to the plugins setup method. | +| [CorePreboot](./kibana-plugin-core-server.corepreboot.md) | Context passed to the setup method of preboot plugins. | +| [CoreSetup](./kibana-plugin-core-server.coresetup.md) | Context passed to the setup method of standard plugins. | | [CoreStart](./kibana-plugin-core-server.corestart.md) | Context passed to the plugins start method. | | [CoreStatus](./kibana-plugin-core-server.corestatus.md) | Status of core services. | | [CountResponse](./kibana-plugin-core-server.countresponse.md) | | @@ -73,6 +75,8 @@ The plugin integrates with the core system via lifecycle events: `setup` | [DeprecationSettings](./kibana-plugin-core-server.deprecationsettings.md) | UiSettings deprecation field options. | | [DeprecationsServiceSetup](./kibana-plugin-core-server.deprecationsservicesetup.md) | The deprecations service provides a way for the Kibana platform to communicate deprecated features and configs with its users. These deprecations are only communicated if the deployment is using these features. Allowing for a user tailored experience for upgrading the stack version.The Deprecation service is consumed by the upgrade assistant to assist with the upgrade experience.If a deprecated feature can be resolved without manual user intervention. Using correctiveActions.api allows the Upgrade Assistant to use this api to correct the deprecation upon a user trigger. | | [DiscoveredPlugin](./kibana-plugin-core-server.discoveredplugin.md) | Small container object used to expose information about discovered plugins that may or may not have been started. | +| [ElasticsearchConfigPreboot](./kibana-plugin-core-server.elasticsearchconfigpreboot.md) | A limited set of Elasticsearch configuration entries exposed to the preboot plugins at setup. | +| [ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) | | | [ElasticsearchServiceSetup](./kibana-plugin-core-server.elasticsearchservicesetup.md) | | | [ElasticsearchServiceStart](./kibana-plugin-core-server.elasticsearchservicestart.md) | | | [ElasticsearchStatusMeta](./kibana-plugin-core-server.elasticsearchstatusmeta.md) | | @@ -87,6 +91,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [HttpResourcesServiceToolkit](./kibana-plugin-core-server.httpresourcesservicetoolkit.md) | Extended set of [KibanaResponseFactory](./kibana-plugin-core-server.kibanaresponsefactory.md) helpers used to respond with HTML or JS resource. | | [HttpResponseOptions](./kibana-plugin-core-server.httpresponseoptions.md) | HTTP response parameters | | [HttpServerInfo](./kibana-plugin-core-server.httpserverinfo.md) | Information about what hostname, port, and protocol the server process is running on. Note that this may not match the URL that end-users access Kibana at. For the public URL, see [BasePath.publicBaseUrl](./kibana-plugin-core-server.basepath.publicbaseurl.md). | +| [HttpServicePreboot](./kibana-plugin-core-server.httpservicepreboot.md) | Kibana HTTP Service provides an abstraction to work with the HTTP stack at the preboot stage. This functionality allows Kibana to serve user requests even before Kibana becomes fully operational. Only Core and preboot plugins can define HTTP routes at this stage. | | [HttpServiceSetup](./kibana-plugin-core-server.httpservicesetup.md) | Kibana HTTP Service provides own abstraction for work with HTTP stack. Plugins don't have direct access to hapi server and its primitives anymore. Moreover, plugins shouldn't rely on the fact that HTTP Service uses one or another library under the hood. This gives the platform flexibility to upgrade or changing our internal HTTP stack without breaking plugins. If the HTTP Service lacks functionality you need, we are happy to discuss and support your needs. | | [HttpServiceStart](./kibana-plugin-core-server.httpservicestart.md) | | | [I18nServiceSetup](./kibana-plugin-core-server.i18nservicesetup.md) | | @@ -128,10 +133,12 @@ The plugin integrates with the core system via lifecycle events: `setup` | [OpsOsMetrics](./kibana-plugin-core-server.opsosmetrics.md) | OS related metrics | | [OpsProcessMetrics](./kibana-plugin-core-server.opsprocessmetrics.md) | Process related metrics | | [OpsServerMetrics](./kibana-plugin-core-server.opsservermetrics.md) | server related metrics | -| [Plugin](./kibana-plugin-core-server.plugin.md) | The interface that should be returned by a PluginInitializer. | +| [Plugin](./kibana-plugin-core-server.plugin.md) | The interface that should be returned by a PluginInitializer for a standard plugin. | | [PluginConfigDescriptor](./kibana-plugin-core-server.pluginconfigdescriptor.md) | Describes a plugin configuration properties. | | [PluginInitializerContext](./kibana-plugin-core-server.plugininitializercontext.md) | Context that's available to plugins during initialization stage. | | [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) | Describes the set of required and optional properties plugin can define in its mandatory JSON manifest file. | +| [PrebootPlugin](./kibana-plugin-core-server.prebootplugin.md) | The interface that should be returned by a PluginInitializer for a preboot plugin. | +| [PrebootServicePreboot](./kibana-plugin-core-server.prebootservicepreboot.md) | Kibana Preboot Service allows to control the boot flow of Kibana. Preboot plugins can use it to hold the boot until certain condition is met. | | [RegisterDeprecationsConfig](./kibana-plugin-core-server.registerdeprecationsconfig.md) | | | [RequestHandlerContext](./kibana-plugin-core-server.requesthandlercontext.md) | Plugin specific context passed to a route handler.Provides the following clients and services: - [savedObjects.client](./kibana-plugin-core-server.savedobjectsclient.md) - Saved Objects client which uses the credentials of the incoming request - [savedObjects.typeRegistry](./kibana-plugin-core-server.isavedobjecttyperegistry.md) - Type registry containing all the registered types. - [elasticsearch.client](./kibana-plugin-core-server.iscopedclusterclient.md) - Elasticsearch data client which uses the credentials of the incoming request - [elasticsearch.legacy.client](./kibana-plugin-core-server.legacyscopedclusterclient.md) - The legacy Elasticsearch data client which uses the credentials of the incoming request - [uiSettings.client](./kibana-plugin-core-server.iuisettingsclient.md) - uiSettings client which uses the credentials of the incoming request | | [ResolveCapabilitiesOptions](./kibana-plugin-core-server.resolvecapabilitiesoptions.md) | Defines a set of additional options for the resolveCapabilities method of [CapabilitiesStart](./kibana-plugin-core-server.capabilitiesstart.md). | diff --git a/docs/development/core/server/kibana-plugin-core-server.plugin.md b/docs/development/core/server/kibana-plugin-core-server.plugin.md index d9796202d78785..b1fce06d46f3a7 100644 --- a/docs/development/core/server/kibana-plugin-core-server.plugin.md +++ b/docs/development/core/server/kibana-plugin-core-server.plugin.md @@ -4,7 +4,7 @@ ## Plugin interface -The interface that should be returned by a `PluginInitializer`. +The interface that should be returned by a `PluginInitializer` for a `standard` plugin. Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.plugininitializer.md b/docs/development/core/server/kibana-plugin-core-server.plugininitializer.md index fe55e131065ddd..9b4d1b022db2a9 100644 --- a/docs/development/core/server/kibana-plugin-core-server.plugininitializer.md +++ b/docs/development/core/server/kibana-plugin-core-server.plugininitializer.md @@ -9,5 +9,5 @@ The `plugin` export at the root of a plugin's `server` directory should conform Signature: ```typescript -export declare type PluginInitializer = (core: PluginInitializerContext) => Plugin | AsyncPlugin; +export declare type PluginInitializer = (core: PluginInitializerContext) => Plugin | PrebootPlugin | AsyncPlugin; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.env.md b/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.env.md index 76e4f222f02285..534f532850587e 100644 --- a/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.env.md +++ b/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.env.md @@ -11,5 +11,6 @@ env: { mode: EnvironmentMode; packageInfo: Readonly; instanceUuid: string; + configs: readonly string[]; }; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.md b/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.md index 90a19d53bd5e1e..9bc9d6d83674c3 100644 --- a/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.md +++ b/docs/development/core/server/kibana-plugin-core-server.plugininitializercontext.md @@ -17,7 +17,7 @@ export interface PluginInitializerContext | Property | Type | Description | | --- | --- | --- | | [config](./kibana-plugin-core-server.plugininitializercontext.config.md) | {
legacy: {
globalConfig$: Observable<SharedGlobalConfig>;
get: () => SharedGlobalConfig;
};
create: <T = ConfigSchema>() => Observable<T>;
get: <T = ConfigSchema>() => T;
} | Accessors for the plugin's configuration | -| [env](./kibana-plugin-core-server.plugininitializercontext.env.md) | {
mode: EnvironmentMode;
packageInfo: Readonly<PackageInfo>;
instanceUuid: string;
} | | +| [env](./kibana-plugin-core-server.plugininitializercontext.env.md) | {
mode: EnvironmentMode;
packageInfo: Readonly<PackageInfo>;
instanceUuid: string;
configs: readonly string[];
} | | | [logger](./kibana-plugin-core-server.plugininitializercontext.logger.md) | LoggerFactory | instance already bound to the plugin's logging context | | [opaqueId](./kibana-plugin-core-server.plugininitializercontext.opaqueid.md) | PluginOpaqueId | | diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md index b3e20bc7ed693a..f8d4c3f1b9d159 100644 --- a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md @@ -31,6 +31,7 @@ Should never be used in code outside of Core but is exported for documentation p | [requiredPlugins](./kibana-plugin-core-server.pluginmanifest.requiredplugins.md) | readonly PluginName[] | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. | | [server](./kibana-plugin-core-server.pluginmanifest.server.md) | boolean | Specifies whether plugin includes some server-side specific functionality. | | [serviceFolders](./kibana-plugin-core-server.pluginmanifest.servicefolders.md) | readonly string[] | Only used for the automatically generated API documentation. Specifying service folders will cause your plugin API reference to be broken up into sub sections. | +| [type](./kibana-plugin-core-server.pluginmanifest.type.md) | PluginType | Type of the plugin, defaults to standard. | | [ui](./kibana-plugin-core-server.pluginmanifest.ui.md) | boolean | Specifies whether plugin includes some client/browser specific functionality that should be included into client bundle via public/ui_plugin.js file. | | [version](./kibana-plugin-core-server.pluginmanifest.version.md) | string | Version of the plugin. | diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.type.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.type.md new file mode 100644 index 00000000000000..6e82132919f6dc --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.type.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) > [type](./kibana-plugin-core-server.pluginmanifest.type.md) + +## PluginManifest.type property + +Type of the plugin, defaults to `standard`. + +Signature: + +```typescript +readonly type: PluginType; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.plugintype.md b/docs/development/core/server/kibana-plugin-core-server.plugintype.md new file mode 100644 index 00000000000000..e4a252a3929499 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.plugintype.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginType](./kibana-plugin-core-server.plugintype.md) + +## PluginType enum + + +Signature: + +```typescript +export declare enum PluginType +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| preboot | "preboot" | Preboot plugins are special-purpose plugins that only function during preboot stage. | +| standard | "standard" | Standard plugins are plugins that start to function as soon as Kibana is fully booted and are active until it shuts down. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.prebootplugin.md b/docs/development/core/server/kibana-plugin-core-server.prebootplugin.md new file mode 100644 index 00000000000000..df851daab78068 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.prebootplugin.md @@ -0,0 +1,21 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PrebootPlugin](./kibana-plugin-core-server.prebootplugin.md) + +## PrebootPlugin interface + +The interface that should be returned by a `PluginInitializer` for a `preboot` plugin. + +Signature: + +```typescript +export interface PrebootPlugin +``` + +## Methods + +| Method | Description | +| --- | --- | +| [setup(core, plugins)](./kibana-plugin-core-server.prebootplugin.setup.md) | | +| [stop()](./kibana-plugin-core-server.prebootplugin.stop.md) | | + diff --git a/docs/development/core/server/kibana-plugin-core-server.prebootplugin.setup.md b/docs/development/core/server/kibana-plugin-core-server.prebootplugin.setup.md new file mode 100644 index 00000000000000..0ee2a26293e980 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.prebootplugin.setup.md @@ -0,0 +1,23 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PrebootPlugin](./kibana-plugin-core-server.prebootplugin.md) > [setup](./kibana-plugin-core-server.prebootplugin.setup.md) + +## PrebootPlugin.setup() method + +Signature: + +```typescript +setup(core: CorePreboot, plugins: TPluginsSetup): TSetup; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| core | CorePreboot | | +| plugins | TPluginsSetup | | + +Returns: + +`TSetup` + diff --git a/docs/development/core/server/kibana-plugin-core-server.prebootplugin.stop.md b/docs/development/core/server/kibana-plugin-core-server.prebootplugin.stop.md new file mode 100644 index 00000000000000..89566b2ae66878 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.prebootplugin.stop.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PrebootPlugin](./kibana-plugin-core-server.prebootplugin.md) > [stop](./kibana-plugin-core-server.prebootplugin.stop.md) + +## PrebootPlugin.stop() method + +Signature: + +```typescript +stop?(): void; +``` +Returns: + +`void` + diff --git a/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.holdsetupuntilresolved.md b/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.holdsetupuntilresolved.md new file mode 100644 index 00000000000000..7f158b46d1f064 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.holdsetupuntilresolved.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PrebootServicePreboot](./kibana-plugin-core-server.prebootservicepreboot.md) > [holdSetupUntilResolved](./kibana-plugin-core-server.prebootservicepreboot.holdsetupuntilresolved.md) + +## PrebootServicePreboot.holdSetupUntilResolved property + +Registers a `Promise` as a precondition before Kibana can proceed to `setup`. This method can be invoked multiple times and from multiple `preboot` plugins. Kibana will proceed to `setup` only when all registered `Promises` instances are resolved, or it will shut down if any of them is rejected. + +Signature: + +```typescript +readonly holdSetupUntilResolved: (reason: string, promise: Promise<{ + shouldReloadConfig: boolean; + } | undefined>) => void; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.issetuponhold.md b/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.issetuponhold.md new file mode 100644 index 00000000000000..1ba079da032081 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.issetuponhold.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PrebootServicePreboot](./kibana-plugin-core-server.prebootservicepreboot.md) > [isSetupOnHold](./kibana-plugin-core-server.prebootservicepreboot.issetuponhold.md) + +## PrebootServicePreboot.isSetupOnHold property + +Indicates whether Kibana is currently on hold and cannot proceed to `setup` yet. + +Signature: + +```typescript +readonly isSetupOnHold: () => boolean; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.md b/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.md new file mode 100644 index 00000000000000..bf503499b6298d --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.prebootservicepreboot.md @@ -0,0 +1,45 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PrebootServicePreboot](./kibana-plugin-core-server.prebootservicepreboot.md) + +## PrebootServicePreboot interface + +Kibana Preboot Service allows to control the boot flow of Kibana. Preboot plugins can use it to hold the boot until certain condition is met. + +Signature: + +```typescript +export interface PrebootServicePreboot +``` + +## Example + +A plugin can supply a `Promise` to a `holdSetupUntilResolved` method to signal Kibana to initialize and start `standard` plugins only after this `Promise` is resolved. If `Promise` is rejected, Kibana will shut down. + +```ts +core.preboot.holdSetupUntilResolved('Just waiting for 5 seconds', + new Promise((resolve) => { + setTimeout(resolve, 5000); + }) +); + +``` +If the supplied `Promise` resolves to an object with the `shouldReloadConfig` property set to `true`, Kibana will also reload its configuration from disk. + +```ts +let completeSetup: (result: { shouldReloadConfig: boolean }) => void; +core.preboot.holdSetupUntilResolved('Just waiting for 5 seconds before reloading configuration', + new Promise<{ shouldReloadConfig: boolean }>((resolve) => { + setTimeout(() => resolve({ shouldReloadConfig: true }), 5000); + }) +); + +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [holdSetupUntilResolved](./kibana-plugin-core-server.prebootservicepreboot.holdsetupuntilresolved.md) | (reason: string, promise: Promise<{
shouldReloadConfig: boolean;
} | undefined>) => void | Registers a Promise as a precondition before Kibana can proceed to setup. This method can be invoked multiple times and from multiple preboot plugins. Kibana will proceed to setup only when all registered Promises instances are resolved, or it will shut down if any of them is rejected. | +| [isSetupOnHold](./kibana-plugin-core-server.prebootservicepreboot.issetuponhold.md) | () => boolean | Indicates whether Kibana is currently on hold and cannot proceed to setup yet. | + diff --git a/examples/preboot_example/README.md b/examples/preboot_example/README.md new file mode 100644 index 00000000000000..0f3ab6c6eae2a6 --- /dev/null +++ b/examples/preboot_example/README.md @@ -0,0 +1,3 @@ +# `prebootExample` plugin + +The example of the `preboot` plugin. \ No newline at end of file diff --git a/examples/preboot_example/kibana.json b/examples/preboot_example/kibana.json new file mode 100644 index 00000000000000..b8b5ceb1a9c6cf --- /dev/null +++ b/examples/preboot_example/kibana.json @@ -0,0 +1,16 @@ +{ + "id": "prebootExample", + "owner": { + "name": "Core", + "githubTeam": "kibana-core" + }, + "description": "The example of the `preboot` plugin.", + "version": "8.0.0", + "kibanaVersion": "kibana", + "configPath": ["prebootExample"], + "type": "preboot", + "server": true, + "ui": true, + "requiredPlugins": [], + "requiredBundles": [] +} diff --git a/examples/preboot_example/public/app.tsx b/examples/preboot_example/public/app.tsx new file mode 100644 index 00000000000000..364b7d5bfe8d32 --- /dev/null +++ b/examples/preboot_example/public/app.tsx @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + EuiButton, + EuiCodeBlock, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiPageTemplate, + EuiPanel, + EuiText, +} from '@elastic/eui'; +import React, { useEffect, useState } from 'react'; +import type { HttpSetup, IHttpFetchError } from 'src/core/public'; + +export const App = ({ http, token }: { http: HttpSetup; token?: string }) => { + const onCompleteSetup = async ({ shouldReloadConfig }: { shouldReloadConfig: boolean }) => { + await http + .post('/api/preboot/complete_setup', { + body: JSON.stringify({ shouldReloadConfig }), + }) + .then(() => { + setTimeout(() => { + window.location.href = '/'; + }, 5000); + }); + }; + + const onWriteToken = async () => { + await http.post('/api/preboot/write_config', { body: JSON.stringify(configKeyValue) }); + }; + + const onConnect = async () => { + await http + .post('/api/preboot/connect_to_es', { body: JSON.stringify(elasticsearchConfig) }) + .then( + (response) => setConnectResponse(JSON.stringify(response)), + (err: IHttpFetchError) => setConnectResponse(err?.body?.message || 'ERROR') + ); + }; + + const [configKeyValue, setConfigKeyValue] = useState<{ key: string; value: string }>({ + key: '', + value: '', + }); + + const [elasticsearchConfig, setElasticsearchConfig] = useState<{ + host: string; + username: string; + password: string; + }>({ + host: 'http://localhost:9200', + username: 'kibana_system', + password: '', + }); + + const [connectResponse, setConnectResponse] = useState(null); + + const [isSetupModeActive, setIsSetupModeActive] = useState(false); + useEffect(() => { + http.get<{ isSetupModeActive: boolean }>('/api/preboot/state').then( + (response) => setIsSetupModeActive(response.isSetupModeActive), + (err: IHttpFetchError) => setIsSetupModeActive(false) + ); + }, [http]); + + if (!isSetupModeActive) { + return ( + + + Kibana server is not ready yet. + + + ); + } + + return ( + + + + + + + { + setConfigKeyValue({ ...configKeyValue, key: e.target.value }); + }} + /> + + + { + setConfigKeyValue({ ...configKeyValue, value: e.target.value }); + }} + /> + + + + Write config + + + + + + + + Token from config: {token} + + + onCompleteSetup({ shouldReloadConfig: true })} + > + Reload config and proceed to `setup` + + + + onCompleteSetup({ shouldReloadConfig: false })} + > + DO NOT reload config and proceed to `setup` + + + + + + + + { + setElasticsearchConfig({ ...elasticsearchConfig, host: e.target.value }); + }} + /> + + + { + setElasticsearchConfig({ + ...elasticsearchConfig, + username: e.target.value, + }); + }} + /> + + + { + setElasticsearchConfig({ + ...elasticsearchConfig, + password: e.target.value, + }); + }} + /> + + + + Connect + + + + + + + {connectResponse ?? ''} + + + + + + ); +}; diff --git a/examples/preboot_example/public/config.ts b/examples/preboot_example/public/config.ts new file mode 100644 index 00000000000000..fc91296ce37249 --- /dev/null +++ b/examples/preboot_example/public/config.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface ConfigType { + token?: string; +} diff --git a/examples/preboot_example/public/index.ts b/examples/preboot_example/public/index.ts new file mode 100644 index 00000000000000..7859758c274f3e --- /dev/null +++ b/examples/preboot_example/public/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PluginInitializerContext } from 'kibana/public'; +import { PrebootExamplePlugin } from './plugin'; + +export const plugin = (initializerContext: PluginInitializerContext) => + new PrebootExamplePlugin(initializerContext); diff --git a/examples/preboot_example/public/plugin.tsx b/examples/preboot_example/public/plugin.tsx new file mode 100644 index 00000000000000..1db38e2240a869 --- /dev/null +++ b/examples/preboot_example/public/plugin.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core/public'; +import React from 'react'; +import ReactDOM from 'react-dom'; +import { App } from './app'; +import { ConfigType } from './config'; + +export class PrebootExamplePlugin implements Plugin { + #config: ConfigType; + constructor(initializerContext: PluginInitializerContext) { + this.#config = initializerContext.config.get(); + } + + public setup(core: CoreSetup) { + core.application.register({ + id: 'prebootExample', + title: 'Preboot Example', + appRoute: '/', + chromeless: true, + mount: (params) => { + ReactDOM.render(, params.element); + return () => ReactDOM.unmountComponentAtNode(params.element); + }, + }); + } + + public start(core: CoreStart) {} +} diff --git a/examples/preboot_example/server/config.ts b/examples/preboot_example/server/config.ts new file mode 100644 index 00000000000000..db39f985bca30e --- /dev/null +++ b/examples/preboot_example/server/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import { schema } from '@kbn/config-schema'; + +export type ConfigType = TypeOf; + +export const ConfigSchema = schema.object({ + enabled: schema.boolean({ defaultValue: false }), + token: schema.maybe(schema.string()), + skipSetup: schema.boolean({ defaultValue: false }), +}); diff --git a/examples/preboot_example/server/index.ts b/examples/preboot_example/server/index.ts new file mode 100644 index 00000000000000..0377250d0cf500 --- /dev/null +++ b/examples/preboot_example/server/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor, PluginInitializerContext } from 'src/core/server'; + +import { ConfigSchema } from './config'; +import { PrebootExamplePlugin } from './plugin'; + +export const config: PluginConfigDescriptor> = { + schema: ConfigSchema, + exposeToBrowser: { token: true }, +}; + +export const plugin = (context: PluginInitializerContext) => new PrebootExamplePlugin(context); diff --git a/examples/preboot_example/server/plugin.ts b/examples/preboot_example/server/plugin.ts new file mode 100644 index 00000000000000..a3c1e9d199143c --- /dev/null +++ b/examples/preboot_example/server/plugin.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; +import type { CorePreboot, PrebootPlugin, PluginInitializerContext } from 'src/core/server'; +import fs from 'fs/promises'; +import { errors } from '@elastic/elasticsearch'; +import Boom from '@hapi/boom'; +import type { ConfigType } from './config'; + +export function getDetailedErrorMessage(error: any): string { + if (error instanceof errors.ResponseError) { + return JSON.stringify(error.body); + } + + if (Boom.isBoom(error)) { + return JSON.stringify(error.output.payload); + } + + return error.message; +} + +export class PrebootExamplePlugin implements PrebootPlugin { + readonly #initializerContext: PluginInitializerContext; + constructor(initializerContext: PluginInitializerContext) { + this.#initializerContext = initializerContext; + } + + public setup(core: CorePreboot) { + const { skipSetup } = this.#initializerContext.config.get(); + let completeSetup: (result: { shouldReloadConfig: boolean }) => void; + + core.http.registerRoutes('', (prebootRouter) => { + prebootRouter.get( + { + path: '/api/preboot/state', + validate: false, + options: { authRequired: false }, + }, + (_, request, response) => { + const isSetupModeActive = !skipSetup && core.preboot.isSetupOnHold(); + return response.ok({ body: { isSetupModeActive } }); + } + ); + if (skipSetup) { + return; + } + + prebootRouter.post( + { + path: '/api/preboot/complete_setup', + validate: { + body: schema.object({ shouldReloadConfig: schema.boolean() }), + }, + options: { authRequired: false }, + }, + (_, request, response) => { + completeSetup({ shouldReloadConfig: request.body.shouldReloadConfig }); + return response.noContent(); + } + ); + + prebootRouter.post( + { + path: '/api/preboot/write_config', + validate: { + body: schema.object({ key: schema.string(), value: schema.string() }), + }, + options: { authRequired: false }, + }, + async (_, request, response) => { + const configPath = this.#initializerContext.env.configs.find((path) => + path.includes('dev') + ); + + if (!configPath) { + return response.customError({ statusCode: 500, body: 'Cannot find dev config.' }); + } + + await fs.appendFile(configPath, `${request.body.key}: ${request.body.value}\n`); + return response.noContent(); + } + ); + + prebootRouter.post( + { + path: '/api/preboot/connect_to_es', + validate: { + body: schema.object({ + host: schema.string(), + username: schema.string(), + password: schema.string(), + }), + }, + options: { authRequired: false }, + }, + async (_, request, response) => { + const esClient = core.elasticsearch.createClient('data', { + hosts: [request.body.host], + }); + + const scopedClient = esClient.asScoped({ + headers: { + authorization: `Basic ${Buffer.from( + `${request.body.username}:${request.body.password}` + ).toString('base64')}`, + }, + }); + + try { + return response.ok({ + body: (await scopedClient.asCurrentUser.security.authenticate()).body, + }); + } catch (err) { + return response.customError({ statusCode: 500, body: getDetailedErrorMessage(err) }); + } + } + ); + + core.preboot.holdSetupUntilResolved( + 'Elasticsearch connection is not set up', + new Promise<{ shouldReloadConfig: boolean }>((resolve) => { + completeSetup = resolve; + }) + ); + }); + } + + public stop() {} +} diff --git a/examples/preboot_example/tsconfig.json b/examples/preboot_example/tsconfig.json new file mode 100644 index 00000000000000..d18953eadf3307 --- /dev/null +++ b/examples/preboot_example/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": ["public/**/*", "server/**/*"], + "references": [{ "path": "../../src/core/tsconfig.json" }] +} diff --git a/packages/kbn-config/src/config_service.test.ts b/packages/kbn-config/src/config_service.test.ts index b1b622381abb1c..d09c61a1c21105 100644 --- a/packages/kbn-config/src/config_service.test.ts +++ b/packages/kbn-config/src/config_service.test.ts @@ -37,6 +37,7 @@ const getRawConfigProvider = (rawConfig: Record) => beforeEach(() => { logger = loggerMock.create(); + mockApplyDeprecations.mockClear(); }); test('returns config at path as observable', async () => { @@ -485,6 +486,16 @@ test('does not log warnings for silent deprecations during validation', async () expect(loggerMock.collect(logger).warn).toMatchInlineSnapshot(`Array []`); }); +test('does not log warnings during validation if specifically requested', async () => { + const configService = new ConfigService(getRawConfigProvider({}), defaultEnv, logger); + loggerMock.clear(logger); + + await configService.validate({ logDeprecations: false }); + + expect(mockApplyDeprecations).not.toHaveBeenCalled(); + expect(loggerMock.collect(logger).warn).toMatchInlineSnapshot(`Array []`); +}); + describe('atPathSync', () => { test('returns the value at path', async () => { const rawConfig = getRawConfigProvider({ key: 'foo' }); diff --git a/packages/kbn-config/src/config_service.ts b/packages/kbn-config/src/config_service.ts index a80680bd46dfca..514992891ad1b0 100644 --- a/packages/kbn-config/src/config_service.ts +++ b/packages/kbn-config/src/config_service.ts @@ -29,6 +29,14 @@ import { LegacyObjectToConfigAdapter } from './legacy'; /** @internal */ export type IConfigService = PublicMethodsOf; +/** @internal */ +export interface ConfigValidateParameters { + /** + * Indicates whether config deprecations should be logged during validation. + */ + logDeprecations: boolean; +} + /** @internal */ export class ConfigService { private readonly log: Logger; @@ -111,13 +119,16 @@ export class ConfigService { * * This must be done after every schemas and deprecation providers have been registered. */ - public async validate() { + public async validate(params: ConfigValidateParameters = { logDeprecations: true }) { const namespaces = [...this.schemas.keys()]; for (let i = 0; i < namespaces.length; i++) { await this.getValidatedConfigAtPath$(namespaces[i]).pipe(first()).toPromise(); } - await this.logDeprecation(); + if (params.logDeprecations) { + await this.logDeprecation(); + } + this.validated = true; } diff --git a/packages/kbn-config/src/index.ts b/packages/kbn-config/src/index.ts index 294caba4e7048b..08cf12343f459e 100644 --- a/packages/kbn-config/src/index.ts +++ b/packages/kbn-config/src/index.ts @@ -25,7 +25,7 @@ export { getConfigFromFiles, } from './raw'; -export { ConfigService, IConfigService } from './config_service'; +export { ConfigService, IConfigService, ConfigValidateParameters } from './config_service'; export { Config, ConfigPath, isConfigPath, hasConfigPathIntersection } from './config'; export { ObjectToConfigAdapter } from './object_to_config_adapter'; export { CliArgs, Env, RawPackageInfo } from './env'; diff --git a/src/core/TESTING.md b/src/core/TESTING.md index 10ead1ea8cfe22..5e0913e7a9d200 100644 --- a/src/core/TESTING.md +++ b/src/core/TESTING.md @@ -337,6 +337,7 @@ describe('myPlugin', () => { let root: ReturnType; beforeAll(async () => { root = kbnTestServer.createRoot(); + await root.preboot(); await root.setup(); await root.start(); }, 30000); @@ -382,6 +383,7 @@ describe('myPlugin', () => { let root: ReturnType; beforeAll(async () => { root = kbnTestServer.createRoot(); + await root.preboot(); await root.setup(); await root.start(); }, 30000); diff --git a/src/core/public/plugins/plugin.test.ts b/src/core/public/plugins/plugin.test.ts index ef919018f120b3..94c88f732f4e19 100644 --- a/src/core/public/plugins/plugin.test.ts +++ b/src/core/public/plugins/plugin.test.ts @@ -8,7 +8,7 @@ import { mockInitializer, mockPlugin, mockPluginReader } from './plugin.test.mocks'; -import { DiscoveredPlugin } from '../../server'; +import { DiscoveredPlugin, PluginType } from '../../server'; import { coreMock } from '../mocks'; import { PluginWrapper } from './plugin'; @@ -19,6 +19,7 @@ function createManifest( return { id, version: 'some-version', + type: PluginType.standard, configPath: ['path'], requiredPlugins: required, optionalPlugins: optional, diff --git a/src/core/public/plugins/plugins_service.test.ts b/src/core/public/plugins/plugins_service.test.ts index d62a4bcdd1e510..3f23889c57de6f 100644 --- a/src/core/public/plugins/plugins_service.test.ts +++ b/src/core/public/plugins/plugins_service.test.ts @@ -13,7 +13,7 @@ import { mockPluginInitializerProvider, } from './plugins_service.test.mocks'; -import { PluginName } from 'src/core/server'; +import { PluginName, PluginType } from 'src/core/server'; import { coreMock } from '../mocks'; import { PluginsService, @@ -60,6 +60,7 @@ function createManifest( return { id, version: 'some-version', + type: PluginType.standard, configPath: ['path'], requiredPlugins: required, optionalPlugins: optional, diff --git a/src/core/server/bootstrap.ts b/src/core/server/bootstrap.ts index 18a5eceb1b2d32..5131defc934616 100644 --- a/src/core/server/bootstrap.ts +++ b/src/core/server/bootstrap.ts @@ -59,9 +59,9 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot reloadConfiguration(); }); - function reloadConfiguration() { + function reloadConfiguration(reason = 'SIGHUP signal received') { const cliLogger = root.logger.get('cli'); - cliLogger.info('Reloading Kibana configuration due to SIGHUP.', { tags: ['config'] }); + cliLogger.info(`Reloading Kibana configuration (reason: ${reason}).`, { tags: ['config'] }); try { rawConfigService.reloadConfig(); @@ -69,7 +69,7 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot return shutdown(err); } - cliLogger.info('Reloaded Kibana configuration due to SIGHUP.', { tags: ['config'] }); + cliLogger.info(`Reloaded Kibana configuration (reason: ${reason}).`, { tags: ['config'] }); } process.on('SIGINT', () => shutdown()); @@ -81,11 +81,28 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot } try { + const { preboot } = await root.preboot(); + + // If setup is on hold then preboot server is supposed to serve user requests and we can let + // dev parent process know that we are ready for dev mode. + const isSetupOnHold = preboot.isSetupOnHold(); + if (process.send && isSetupOnHold) { + process.send(['SERVER_LISTENING']); + } + + if (isSetupOnHold) { + root.logger.get().info('Holding setup until preboot stage is completed.'); + const { shouldReloadConfig } = await preboot.waitUntilCanSetup(); + if (shouldReloadConfig) { + await reloadConfiguration('configuration might have changed during preboot stage'); + } + } + await root.setup(); await root.start(); - // notify parent process know when we are ready for dev mode. - if (process.send) { + // Notify parent process if we haven't done that yet during preboot stage. + if (process.send && !isSetupOnHold) { process.send(['SERVER_LISTENING']); } } catch (err) { diff --git a/src/core/server/capabilities/capabilities_service.mock.ts b/src/core/server/capabilities/capabilities_service.mock.ts index 0ba3446a772706..1af10b3ad981e2 100644 --- a/src/core/server/capabilities/capabilities_service.mock.ts +++ b/src/core/server/capabilities/capabilities_service.mock.ts @@ -36,6 +36,7 @@ const createCapabilitiesMock = (): Capabilities => { type CapabilitiesServiceContract = PublicMethodsOf; const createMock = () => { const mocked: jest.Mocked = { + preboot: jest.fn(), setup: jest.fn().mockReturnValue(createSetupContractMock()), start: jest.fn().mockReturnValue(createStartContractMock()), }; diff --git a/src/core/server/capabilities/capabilities_service.test.ts b/src/core/server/capabilities/capabilities_service.test.ts index 8f9627a64b082c..0476f844d011f6 100644 --- a/src/core/server/capabilities/capabilities_service.test.ts +++ b/src/core/server/capabilities/capabilities_service.test.ts @@ -6,7 +6,11 @@ * Side Public License, v 1. */ -import { httpServiceMock, InternalHttpServiceSetupMock } from '../http/http_service.mock'; +import { + httpServiceMock, + InternalHttpServicePrebootMock, + InternalHttpServiceSetupMock, +} from '../http/http_service.mock'; import { mockRouter, RouterMock } from '../http/router/router.mock'; import { CapabilitiesService, CapabilitiesSetup } from './capabilities_service'; import { mockCoreContext } from '../core_context.mock'; @@ -24,6 +28,31 @@ describe('CapabilitiesService', () => { service = new CapabilitiesService(mockCoreContext.create()); }); + describe('#preboot()', () => { + let httpPreboot: InternalHttpServicePrebootMock; + beforeEach(() => { + httpPreboot = httpServiceMock.createInternalPrebootContract(); + service.preboot({ http: httpPreboot }); + }); + + it('registers the capabilities routes', async () => { + expect(httpPreboot.registerRoutes).toHaveBeenCalledWith('', expect.any(Function)); + expect(httpPreboot.registerRoutes).toHaveBeenCalledTimes(1); + + const [[, callback]] = httpPreboot.registerRoutes.mock.calls; + callback(router); + + expect(router.post).toHaveBeenCalledTimes(1); + expect(router.post).toHaveBeenCalledWith( + expect.objectContaining({ + path: '/api/core/capabilities', + options: { authRequired: 'optional' }, + }), + expect.any(Function) + ); + }); + }); + describe('#setup()', () => { beforeEach(() => { setup = service.setup({ http }); diff --git a/src/core/server/capabilities/capabilities_service.ts b/src/core/server/capabilities/capabilities_service.ts index 6088ec29db9985..1166c8f6b48c4d 100644 --- a/src/core/server/capabilities/capabilities_service.ts +++ b/src/core/server/capabilities/capabilities_service.ts @@ -9,7 +9,7 @@ import { Capabilities, CapabilitiesProvider, CapabilitiesSwitcher } from './types'; import { CoreContext } from '../core_context'; import { Logger } from '../logging'; -import { InternalHttpServiceSetup, KibanaRequest } from '../http'; +import { InternalHttpServicePreboot, InternalHttpServiceSetup, KibanaRequest } from '../http'; import { mergeCapabilities } from './merge_capabilities'; import { getCapabilitiesResolver, CapabilitiesResolver } from './resolve_capabilities'; import { registerRoutes } from './routes'; @@ -120,6 +120,10 @@ export interface CapabilitiesStart { ): Promise; } +interface PrebootSetupDeps { + http: InternalHttpServicePreboot; +} + interface SetupDeps { http: InternalHttpServiceSetup; } @@ -149,10 +153,20 @@ export class CapabilitiesService { ); } + public preboot(prebootDeps: PrebootSetupDeps) { + this.logger.debug('Prebooting capabilities service'); + + // The preboot server has no need for real capabilities. + // Returning the un-augmented defaults is sufficient. + prebootDeps.http.registerRoutes('', (router) => { + registerRoutes(router, async () => defaultCapabilities); + }); + } + public setup(setupDeps: SetupDeps): CapabilitiesSetup { this.logger.debug('Setting up capabilities service'); - registerRoutes(setupDeps.http, this.resolveCapabilities); + registerRoutes(setupDeps.http.createRouter(''), this.resolveCapabilities); return { registerProvider: (provider: CapabilitiesProvider) => { diff --git a/src/core/server/capabilities/integration_tests/capabilities_service.test.ts b/src/core/server/capabilities/integration_tests/capabilities_service.test.ts index ac793d960d03b8..2e80fbb9d20c0a 100644 --- a/src/core/server/capabilities/integration_tests/capabilities_service.test.ts +++ b/src/core/server/capabilities/integration_tests/capabilities_service.test.ts @@ -8,7 +8,7 @@ import supertest from 'supertest'; import { REPO_ROOT } from '@kbn/dev-utils'; -import { HttpService, InternalHttpServiceSetup } from '../../http'; +import { HttpService, InternalHttpServicePreboot, InternalHttpServiceSetup } from '../../http'; import { contextServiceMock } from '../../context/context_service.mock'; import { executionContextServiceMock } from '../../execution_context/execution_context_service.mock'; import { loggingSystemMock } from '../../logging/logging_system.mock'; @@ -23,6 +23,7 @@ const env = Env.createDefault(REPO_ROOT, getEnvOptions()); describe('CapabilitiesService', () => { let server: HttpService; + let httpPreboot: InternalHttpServicePreboot; let httpSetup: InternalHttpServiceSetup; let service: CapabilitiesService; @@ -30,6 +31,7 @@ describe('CapabilitiesService', () => { beforeEach(async () => { server = createHttpServer(); + httpPreboot = await server.preboot({ context: contextServiceMock.createPrebootContract() }); httpSetup = await server.setup({ context: contextServiceMock.createSetupContract(), executionContext: executionContextServiceMock.createInternalSetupContract(), @@ -40,6 +42,7 @@ describe('CapabilitiesService', () => { logger: loggingSystemMock.create(), configService: {} as any, }); + await service.preboot({ http: httpPreboot }); serviceSetup = await service.setup({ http: httpSetup }); await server.start(); }); diff --git a/src/core/server/capabilities/routes/index.ts b/src/core/server/capabilities/routes/index.ts index a417dbfeb43a3c..4140d75d1a1a10 100644 --- a/src/core/server/capabilities/routes/index.ts +++ b/src/core/server/capabilities/routes/index.ts @@ -7,10 +7,9 @@ */ import { CapabilitiesResolver } from '../resolve_capabilities'; -import { InternalHttpServiceSetup } from '../../http'; +import { IRouter } from '../../http'; import { registerCapabilitiesRoutes } from './resolve_capabilities'; -export function registerRoutes(http: InternalHttpServiceSetup, resolver: CapabilitiesResolver) { - const router = http.createRouter(''); +export function registerRoutes(router: IRouter, resolver: CapabilitiesResolver) { registerCapabilitiesRoutes(router, resolver); } diff --git a/src/core/server/config/ensure_valid_configuration.test.ts b/src/core/server/config/ensure_valid_configuration.test.ts index f1006f93dbc2d4..372b9d4c0dfad9 100644 --- a/src/core/server/config/ensure_valid_configuration.test.ts +++ b/src/core/server/config/ensure_valid_configuration.test.ts @@ -23,6 +23,20 @@ describe('ensureValidConfiguration', () => { it('returns normally when there is no unused keys and when the config validates', async () => { await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined(); + + expect(configService.validate).toHaveBeenCalledWith(undefined); + }); + + it('forwards parameters to the `validate` method', async () => { + await expect( + ensureValidConfiguration(configService as any, { logDeprecations: false }) + ).resolves.toBeUndefined(); + expect(configService.validate).toHaveBeenCalledWith({ logDeprecations: false }); + + await expect( + ensureValidConfiguration(configService as any, { logDeprecations: true }) + ).resolves.toBeUndefined(); + expect(configService.validate).toHaveBeenCalledWith({ logDeprecations: true }); }); it('throws when config validation fails', async () => { diff --git a/src/core/server/config/ensure_valid_configuration.ts b/src/core/server/config/ensure_valid_configuration.ts index c7a4721b7d2aea..040cb23e079887 100644 --- a/src/core/server/config/ensure_valid_configuration.ts +++ b/src/core/server/config/ensure_valid_configuration.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { ConfigService } from '@kbn/config'; +import { ConfigService, ConfigValidateParameters } from '@kbn/config'; import { CriticalError } from '../errors'; const ignoredPaths = ['dev.', 'elastic.apm.']; @@ -14,9 +14,12 @@ const ignoredPaths = ['dev.', 'elastic.apm.']; const invalidConfigExitCode = 78; const legacyInvalidConfigExitCode = 64; -export async function ensureValidConfiguration(configService: ConfigService) { +export async function ensureValidConfiguration( + configService: ConfigService, + params?: ConfigValidateParameters +) { try { - await configService.validate(); + await configService.validate(params); } catch (e) { throw new CriticalError(e.message, 'InvalidConfig', invalidConfigExitCode, e); } diff --git a/src/core/server/config/integration_tests/config_deprecation.test.ts b/src/core/server/config/integration_tests/config_deprecation.test.ts index 5b672774c515a9..2d86281ce40d62 100644 --- a/src/core/server/config/integration_tests/config_deprecation.test.ts +++ b/src/core/server/config/integration_tests/config_deprecation.test.ts @@ -26,6 +26,7 @@ describe('configuration deprecations', () => { it('should not log deprecation warnings for default configuration that is not one of `logging.verbose`, `logging.quiet` or `logging.silent`', async () => { root = kbnTestServer.createRoot(); + await root.preboot(); await root.setup(); const logs = loggingSystemMock.collect(mockLoggingSystem); @@ -44,6 +45,7 @@ describe('configuration deprecations', () => { }, }); + await root.preboot(); await root.setup(); const logs = loggingSystemMock.collect(mockLoggingSystem); diff --git a/src/core/server/context/context_service.mock.ts b/src/core/server/context/context_service.mock.ts index e705fff2e35a51..c9b1e0e7692e50 100644 --- a/src/core/server/context/context_service.mock.ts +++ b/src/core/server/context/context_service.mock.ts @@ -8,9 +8,16 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; -import { ContextService, ContextSetup } from './context_service'; +import { ContextService, ContextSetup, InternalContextPreboot } from './context_service'; import { contextMock } from './container/context.mock'; +const createPrebootContractMock = (mockContext = {}) => { + const prebootContract: jest.Mocked = { + createContextContainer: jest.fn().mockImplementation(() => contextMock.create(mockContext)), + }; + return prebootContract; +}; + const createSetupContractMock = (mockContext = {}) => { const setupContract: jest.Mocked = { createContextContainer: jest.fn().mockImplementation(() => contextMock.create(mockContext)), @@ -21,13 +28,16 @@ const createSetupContractMock = (mockContext = {}) => { type ContextServiceContract = PublicMethodsOf; const createMock = () => { const mocked: jest.Mocked = { + preboot: jest.fn(), setup: jest.fn(), }; + mocked.preboot.mockReturnValue(createPrebootContractMock()); mocked.setup.mockReturnValue(createSetupContractMock()); return mocked; }; export const contextServiceMock = { create: createMock, + createPrebootContract: createPrebootContractMock, createSetupContract: createSetupContractMock, }; diff --git a/src/core/server/context/context_service.test.ts b/src/core/server/context/context_service.test.ts index 48dc485727251b..8b5a784b1f8a14 100644 --- a/src/core/server/context/context_service.test.ts +++ b/src/core/server/context/context_service.test.ts @@ -14,10 +14,23 @@ import { CoreContext } from '../core_context'; const pluginDependencies = new Map(); describe('ContextService', () => { + describe('#preboot()', () => { + test('createContextContainer returns a new container configured with pluginDependencies', () => { + const coreId = Symbol(); + const service = new ContextService({ coreId } as CoreContext); + const preboot = service.preboot({ pluginDependencies }); + expect(preboot.createContextContainer()).toBeDefined(); + expect(MockContextConstructor).toHaveBeenCalledWith(pluginDependencies, coreId); + }); + }); + describe('#setup()', () => { test('createContextContainer returns a new container configured with pluginDependencies', () => { const coreId = Symbol(); const service = new ContextService({ coreId } as CoreContext); + + service.preboot({ pluginDependencies: new Map() }); + const setup = service.setup({ pluginDependencies }); expect(setup.createContextContainer()).toBeDefined(); expect(MockContextConstructor).toHaveBeenCalledWith(pluginDependencies, coreId); diff --git a/src/core/server/context/context_service.ts b/src/core/server/context/context_service.ts index 68a99ed3d41561..9e77786c1562cb 100644 --- a/src/core/server/context/context_service.ts +++ b/src/core/server/context/context_service.ts @@ -10,6 +10,8 @@ import { PluginOpaqueId } from '../../server'; import { IContextContainer, ContextContainer } from './container'; import { CoreContext } from '../core_context'; +type PrebootDeps = SetupDeps; + interface SetupDeps { pluginDependencies: ReadonlyMap; } @@ -18,7 +20,17 @@ interface SetupDeps { export class ContextService { constructor(private readonly core: CoreContext) {} + public preboot({ pluginDependencies }: PrebootDeps): InternalContextPreboot { + return this.getContextContainerFactory(pluginDependencies); + } + public setup({ pluginDependencies }: SetupDeps): ContextSetup { + return this.getContextContainerFactory(pluginDependencies); + } + + private getContextContainerFactory( + pluginDependencies: ReadonlyMap + ) { return { createContextContainer: () => { return new ContextContainer(pluginDependencies, this.core.coreId); @@ -27,6 +39,9 @@ export class ContextService { } } +/** @internal */ +export type InternalContextPreboot = ContextSetup; + /** * {@inheritdoc IContextContainer} * diff --git a/src/core/server/context/index.ts b/src/core/server/context/index.ts index 84f7ad07da2c3e..d12bafdef7a901 100644 --- a/src/core/server/context/index.ts +++ b/src/core/server/context/index.ts @@ -7,7 +7,7 @@ */ export { ContextService } from './context_service'; -export type { ContextSetup } from './context_service'; +export type { InternalContextPreboot, ContextSetup } from './context_service'; export type { IContextContainer, IContextProvider, diff --git a/src/core/server/core_app/core_app.test.ts b/src/core/server/core_app/core_app.test.ts index ad7af3ac8b84df..f6a9b653ec0345 100644 --- a/src/core/server/core_app/core_app.test.ts +++ b/src/core/server/core_app/core_app.test.ts @@ -9,10 +9,13 @@ import { registerBundleRoutesMock } from './core_app.test.mocks'; import { mockCoreContext } from '../core_context.mock'; -import { coreMock } from '../mocks'; +import { coreMock, httpServerMock } from '../mocks'; import { httpResourcesMock } from '../http_resources/http_resources_service.mock'; import type { UiPlugins } from '../plugins'; +import { PluginType } from '../plugins'; import { CoreApp } from './core_app'; +import { mockRouter } from '../http/router/router.mock'; +import { RequestHandlerContext } from 'kibana/server'; const emptyPlugins = (): UiPlugins => ({ internal: new Map(), @@ -23,11 +26,23 @@ const emptyPlugins = (): UiPlugins => ({ describe('CoreApp', () => { let coreContext: ReturnType; let coreApp: CoreApp; + let internalCorePreboot: ReturnType; + let prebootHTTPResourcesRegistrar: ReturnType; let internalCoreSetup: ReturnType; let httpResourcesRegistrar: ReturnType; beforeEach(() => { coreContext = mockCoreContext.create(); + + internalCorePreboot = coreMock.createInternalPreboot(); + internalCorePreboot.http.registerRoutes.mockImplementation((path, callback) => + callback(mockRouter.create()) + ); + prebootHTTPResourcesRegistrar = httpResourcesMock.createRegistrar(); + internalCorePreboot.httpResources.createRegistrar.mockReturnValue( + prebootHTTPResourcesRegistrar + ); + internalCoreSetup = coreMock.createInternalSetup(); httpResourcesRegistrar = httpResourcesMock.createRegistrar(); internalCoreSetup.httpResources.createRegistrar.mockReturnValue(httpResourcesRegistrar); @@ -72,6 +87,60 @@ describe('CoreApp', () => { }); }); + describe('#preboot', () => { + let prebootUIPlugins: UiPlugins; + beforeEach(() => { + prebootUIPlugins = emptyPlugins(); + prebootUIPlugins.public.set('some-plugin', { + type: PluginType.preboot, + configPath: 'some-plugin', + id: 'some-plugin', + optionalPlugins: [], + requiredBundles: [], + requiredPlugins: [], + }); + }); + it('calls `registerBundleRoutes` with the correct options', () => { + coreApp.preboot(internalCorePreboot, prebootUIPlugins); + + expect(registerBundleRoutesMock).toHaveBeenCalledTimes(1); + expect(registerBundleRoutesMock).toHaveBeenCalledWith({ + uiPlugins: prebootUIPlugins, + router: expect.any(Object), + packageInfo: coreContext.env.packageInfo, + serverBasePath: internalCorePreboot.http.basePath.serverBasePath, + }); + }); + + it('does not call `registerBundleRoutes` if there are no `preboot` UI plugins', () => { + coreApp.preboot(internalCorePreboot, emptyPlugins()); + + expect(registerBundleRoutesMock).not.toHaveBeenCalled(); + }); + + it('main route handles core app rendering', () => { + coreApp.preboot(internalCorePreboot, prebootUIPlugins); + + expect(prebootHTTPResourcesRegistrar.register).toHaveBeenCalledWith( + { + path: '/{path*}', + validate: expect.any(Object), + }, + expect.any(Function) + ); + + const [[, handler]] = prebootHTTPResourcesRegistrar.register.mock.calls; + const mockResponseFactory = httpResourcesMock.createResponseFactory(); + handler( + ({} as unknown) as RequestHandlerContext, + httpServerMock.createKibanaRequest(), + mockResponseFactory + ); + + expect(mockResponseFactory.renderAnonymousCoreApp).toHaveBeenCalled(); + }); + }); + describe('`/app/{id}/{any*}` route', () => { it('is registered with the correct parameters', () => { coreApp.setup(internalCoreSetup, emptyPlugins()); @@ -89,7 +158,7 @@ describe('CoreApp', () => { }); }); - it('calls `registerBundleRoutes` with the correct options', () => { + it('`setup` calls `registerBundleRoutes` with the correct options', () => { const uiPlugins = emptyPlugins(); coreApp.setup(internalCoreSetup, uiPlugins); diff --git a/src/core/server/core_app/core_app.ts b/src/core/server/core_app/core_app.ts index e728cb0b824757..35a7c57b67610e 100644 --- a/src/core/server/core_app/core_app.ts +++ b/src/core/server/core_app/core_app.ts @@ -12,12 +12,25 @@ import { Env } from '@kbn/config'; import { schema } from '@kbn/config-schema'; import { fromRoot } from '@kbn/utils'; -import { InternalCoreSetup } from '../internal_types'; +import { IRouter, IBasePath, IKibanaResponse, KibanaResponseFactory } from '../http'; +import { HttpResources, HttpResourcesServiceToolkit } from '../http_resources'; +import { InternalCorePreboot, InternalCoreSetup } from '../internal_types'; import { CoreContext } from '../core_context'; import { Logger } from '../logging'; import { registerBundleRoutes } from './bundle_routes'; import { UiPlugins } from '../plugins'; +/** @internal */ +interface CommonRoutesParams { + router: IRouter; + httpResources: HttpResources; + basePath: IBasePath; + uiPlugins: UiPlugins; + onResourceNotFound: ( + res: HttpResourcesServiceToolkit & KibanaResponseFactory + ) => Promise; +} + /** @internal */ export class CoreApp { private readonly logger: Logger; @@ -28,12 +41,34 @@ export class CoreApp { this.env = core.env; } + preboot(corePreboot: InternalCorePreboot, uiPlugins: UiPlugins) { + this.logger.debug('Prebooting core app.'); + + // We register app-serving routes only if there are `preboot` plugins that may need them. + if (uiPlugins.public.size > 0) { + this.registerPrebootDefaultRoutes(corePreboot, uiPlugins); + this.registerStaticDirs(corePreboot); + } + } + setup(coreSetup: InternalCoreSetup, uiPlugins: UiPlugins) { this.logger.debug('Setting up core app.'); this.registerDefaultRoutes(coreSetup, uiPlugins); this.registerStaticDirs(coreSetup); } + private registerPrebootDefaultRoutes(corePreboot: InternalCorePreboot, uiPlugins: UiPlugins) { + corePreboot.http.registerRoutes('', (router) => { + this.registerCommonDefaultRoutes({ + basePath: corePreboot.http.basePath, + httpResources: corePreboot.httpResources.createRegistrar(router), + router, + uiPlugins, + onResourceNotFound: (res) => res.renderAnonymousCoreApp(), + }); + }); + } + private registerDefaultRoutes(coreSetup: InternalCoreSetup, uiPlugins: UiPlugins) { const httpSetup = coreSetup.http; const router = httpSetup.createRouter(''); @@ -51,8 +86,55 @@ export class CoreApp { }); }); - // remove trailing slash catch-all - router.get( + this.registerCommonDefaultRoutes({ + basePath: coreSetup.http.basePath, + httpResources: resources, + router, + uiPlugins, + onResourceNotFound: async (res) => res.notFound(), + }); + + resources.register( + { + path: '/app/{id}/{any*}', + validate: false, + options: { + authRequired: true, + }, + }, + async (context, request, response) => { + return response.renderCoreApp(); + } + ); + + const anonymousStatusPage = coreSetup.status.isStatusPageAnonymous(); + resources.register( + { + path: '/status', + validate: false, + options: { + authRequired: !anonymousStatusPage, + }, + }, + async (context, request, response) => { + if (anonymousStatusPage) { + return response.renderAnonymousCoreApp(); + } else { + return response.renderCoreApp(); + } + } + ); + } + + private registerCommonDefaultRoutes({ + router, + basePath, + uiPlugins, + onResourceNotFound, + httpResources, + }: CommonRoutesParams) { + // catch-all route + httpResources.register( { path: '/{path*}', validate: { @@ -66,17 +148,18 @@ export class CoreApp { const { query, params } = req; const { path } = params; if (!path || !path.endsWith('/') || path.startsWith('/')) { - return res.notFound(); + return onResourceNotFound(res); } - const basePath = httpSetup.basePath.get(req); + // remove trailing slash + const requestBasePath = basePath.get(req); let rewrittenPath = path.slice(0, -1); - if (`/${path}`.startsWith(basePath)) { - rewrittenPath = rewrittenPath.substring(basePath.length); + if (`/${path}`.startsWith(requestBasePath)) { + rewrittenPath = rewrittenPath.substring(requestBasePath.length); } const querystring = query ? stringify(query) : undefined; - const url = `${basePath}/${rewrittenPath}${querystring ? `?${querystring}` : ''}`; + const url = `${requestBasePath}/${rewrittenPath}${querystring ? `?${querystring}` : ''}`; return res.redirected({ headers: { @@ -94,45 +177,14 @@ export class CoreApp { router, uiPlugins, packageInfo: this.env.packageInfo, - serverBasePath: coreSetup.http.basePath.serverBasePath, + serverBasePath: basePath.serverBasePath, }); - - resources.register( - { - path: '/app/{id}/{any*}', - validate: false, - options: { - authRequired: true, - }, - }, - async (context, request, response) => { - return response.renderCoreApp(); - } - ); - - const anonymousStatusPage = coreSetup.status.isStatusPageAnonymous(); - resources.register( - { - path: '/status', - validate: false, - options: { - authRequired: !anonymousStatusPage, - }, - }, - async (context, request, response) => { - if (anonymousStatusPage) { - return response.renderAnonymousCoreApp(); - } else { - return response.renderCoreApp(); - } - } - ); } - private registerStaticDirs(coreSetup: InternalCoreSetup) { - coreSetup.http.registerStaticDir('/ui/{path*}', Path.resolve(__dirname, './assets')); + private registerStaticDirs(core: InternalCoreSetup | InternalCorePreboot) { + core.http.registerStaticDir('/ui/{path*}', Path.resolve(__dirname, './assets')); - coreSetup.http.registerStaticDir( + core.http.registerStaticDir( '/node_modules/@kbn/ui-framework/dist/{path*}', fromRoot('node_modules/@kbn/ui-framework/dist') ); diff --git a/src/core/server/core_app/integration_tests/bundle_routes.test.ts b/src/core/server/core_app/integration_tests/bundle_routes.test.ts index 7c50e09b124688..6d37241c2a36fa 100644 --- a/src/core/server/core_app/integration_tests/bundle_routes.test.ts +++ b/src/core/server/core_app/integration_tests/bundle_routes.test.ts @@ -26,12 +26,13 @@ describe('bundle routes', () => { let logger: ReturnType; let fileHashCache: FileHashCache; - beforeEach(() => { + beforeEach(async () => { contextSetup = contextServiceMock.createSetupContract(); logger = loggingSystemMock.create(); fileHashCache = new FileHashCache(); server = createHttpServer({ logger }); + await server.preboot({ context: contextServiceMock.createPrebootContract() }); }); afterEach(async () => { diff --git a/src/core/server/core_app/integration_tests/core_app_routes.test.ts b/src/core/server/core_app/integration_tests/core_app_routes.test.ts index faa1c905afa9d5..a12e9e7d551881 100644 --- a/src/core/server/core_app/integration_tests/core_app_routes.test.ts +++ b/src/core/server/core_app/integration_tests/core_app_routes.test.ts @@ -20,6 +20,7 @@ describe('Core app routes', () => { }, }); + await root.preboot(); await root.setup(); await root.start(); }); diff --git a/src/core/server/core_app/integration_tests/default_route_provider_config.test.ts b/src/core/server/core_app/integration_tests/default_route_provider_config.test.ts index dcb623c1ffc743..2ffe3b9d1e4cc3 100644 --- a/src/core/server/core_app/integration_tests/default_route_provider_config.test.ts +++ b/src/core/server/core_app/integration_tests/default_route_provider_config.test.ts @@ -24,6 +24,7 @@ describe('default route provider', () => { }, }); + await root.preboot(); await root.setup(); await root.start(); }); diff --git a/src/core/server/core_app/integration_tests/static_assets.test.ts b/src/core/server/core_app/integration_tests/static_assets.test.ts index 1c7b7db305b7a9..86da1d94d3fc68 100644 --- a/src/core/server/core_app/integration_tests/static_assets.test.ts +++ b/src/core/server/core_app/integration_tests/static_assets.test.ts @@ -15,6 +15,7 @@ describe('Platform assets', function () { beforeAll(async function () { root = kbnTestServer.createRoot({ plugins: { initialize: false } }); + await root.preboot(); await root.setup(); await root.start(); }); diff --git a/src/core/server/elasticsearch/elasticsearch_service.mock.ts b/src/core/server/elasticsearch/elasticsearch_service.mock.ts index 089028fc329761..0ccc0f51f6abd5 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.mock.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.mock.ts @@ -19,10 +19,16 @@ import { ElasticsearchClientConfig } from './client'; import { legacyClientMock } from './legacy/mocks'; import { ElasticsearchConfig } from './elasticsearch_config'; import { ElasticsearchService } from './elasticsearch_service'; -import { InternalElasticsearchServiceSetup, ElasticsearchStatusMeta } from './types'; +import { + InternalElasticsearchServiceSetup, + ElasticsearchStatusMeta, + ElasticsearchServicePreboot, +} from './types'; import { NodesVersionCompatibility } from './version_check/ensure_es_version'; import { ServiceStatus, ServiceStatusLevels } from '../status'; +type MockedElasticSearchServicePreboot = jest.Mocked; + export interface MockedElasticSearchServiceSetup { legacy: { config$: BehaviorSubject; @@ -38,6 +44,17 @@ type MockedElasticSearchServiceStart = MockedElasticSearchServiceSetup & { >; }; +const createPrebootContractMock = () => { + const prebootContract: MockedElasticSearchServicePreboot = { + config: { hosts: [], credentialsSpecified: false }, + createClient: jest.fn(), + }; + prebootContract.createClient.mockImplementation(() => + elasticsearchClientMock.createCustomClusterClient() + ); + return prebootContract; +}; + const createSetupContractMock = () => { const setupContract: MockedElasticSearchServiceSetup = { legacy: { @@ -73,7 +90,7 @@ const createStartContractMock = () => { return startContract; }; -const createInternalStartContractMock = createStartContractMock; +const createInternalPrebootContractMock = createPrebootContractMock; type MockedInternalElasticSearchServiceSetup = jest.Mocked< InternalElasticsearchServiceSetup & { @@ -102,13 +119,17 @@ const createInternalSetupContractMock = () => { return setupContract; }; +const createInternalStartContractMock = createStartContractMock; + type ElasticsearchServiceContract = PublicMethodsOf; const createMock = () => { const mocked: jest.Mocked = { + preboot: jest.fn(), setup: jest.fn(), start: jest.fn(), stop: jest.fn(), }; + mocked.preboot.mockResolvedValue(createInternalPrebootContractMock()); mocked.setup.mockResolvedValue(createInternalSetupContractMock()); mocked.start.mockResolvedValueOnce(createInternalStartContractMock()); mocked.stop.mockResolvedValue(); @@ -117,6 +138,8 @@ const createMock = () => { export const elasticsearchServiceMock = { create: createMock, + createInternalPreboot: createInternalPrebootContractMock, + createPreboot: createPrebootContractMock, createInternalSetup: createInternalSetupContractMock, createSetup: createSetupContractMock, createInternalStart: createInternalStartContractMock, diff --git a/src/core/server/elasticsearch/elasticsearch_service.test.ts b/src/core/server/elasticsearch/elasticsearch_service.test.ts index 791ae2ab7abaa8..8932a4c73e1f2a 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.test.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.test.ts @@ -16,7 +16,7 @@ import { CoreContext } from '../core_context'; import { loggingSystemMock } from '../logging/logging_system.mock'; import { httpServiceMock } from '../http/http_service.mock'; import { executionContextServiceMock } from '../execution_context/execution_context_service.mock'; -import { ElasticsearchConfig } from './elasticsearch_config'; +import { configSchema, ElasticsearchConfig } from './elasticsearch_config'; import { ElasticsearchService } from './elasticsearch_service'; import { elasticsearchServiceMock } from './elasticsearch_service.mock'; import { elasticsearchClientMock } from './client/mocks'; @@ -31,17 +31,6 @@ const setupDeps = { http: httpServiceMock.createInternalSetupContract(), executionContext: executionContextServiceMock.createInternalSetupContract(), }; -configService.atPath.mockReturnValue( - new BehaviorSubject({ - hosts: ['http://1.2.3.4'], - healthCheck: { - delay: duration(10), - }, - ssl: { - verificationMode: 'none', - }, - } as any) -); let env: Env; let coreContext: CoreContext; @@ -51,10 +40,21 @@ let mockClusterClientInstance: ReturnType; - +let mockConfig$: BehaviorSubject; beforeEach(() => { env = Env.createDefault(REPO_ROOT, getEnvOptions()); + mockConfig$ = new BehaviorSubject({ + hosts: ['http://1.2.3.4'], + healthCheck: { + delay: duration(10), + }, + ssl: { + verificationMode: 'none', + }, + }); + configService.atPath.mockReturnValue(mockConfig$); + coreContext = { coreId: Symbol(), env, logger, configService: configService as any }; elasticsearchService = new ElasticsearchService(coreContext); @@ -69,6 +69,90 @@ beforeEach(() => { afterEach(() => jest.clearAllMocks()); +describe('#preboot', () => { + describe('#config', () => { + it('exposes `hosts`', async () => { + const prebootContract = await elasticsearchService.preboot(); + expect(prebootContract.config).toEqual({ + credentialsSpecified: false, + hosts: ['http://1.2.3.4'], + }); + }); + + it('set `credentialsSpecified` to `true` if `username` is specified', async () => { + mockConfig$.next(configSchema.validate({ username: 'kibana_system' })); + const prebootContract = await elasticsearchService.preboot(); + expect(prebootContract.config.credentialsSpecified).toBe(true); + }); + + it('set `credentialsSpecified` to `true` if `password` is specified', async () => { + mockConfig$.next(configSchema.validate({ password: 'changeme' })); + const prebootContract = await elasticsearchService.preboot(); + expect(prebootContract.config.credentialsSpecified).toBe(true); + }); + + it('set `credentialsSpecified` to `true` if `serviceAccountToken` is specified', async () => { + mockConfig$.next(configSchema.validate({ serviceAccountToken: 'xxxx' })); + const prebootContract = await elasticsearchService.preboot(); + expect(prebootContract.config.credentialsSpecified).toBe(true); + }); + }); + + describe('#createClient', () => { + it('allows to specify config properties', async () => { + const prebootContract = await elasticsearchService.preboot(); + const customConfig = { keepAlive: true }; + const clusterClient = prebootContract.createClient('custom-type', customConfig); + + expect(clusterClient).toBe(mockClusterClientInstance); + + expect(MockClusterClient).toHaveBeenCalledTimes(1); + expect(MockClusterClient.mock.calls[0][0]).toEqual(expect.objectContaining(customConfig)); + }); + + it('creates a new client on each call', async () => { + const prebootContract = await elasticsearchService.preboot(); + + const customConfig = { keepAlive: true }; + + prebootContract.createClient('custom-type', customConfig); + prebootContract.createClient('another-type', customConfig); + + expect(MockClusterClient).toHaveBeenCalledTimes(2); + }); + + it('falls back to elasticsearch default config values if property not specified', async () => { + const prebootContract = await elasticsearchService.preboot(); + + const customConfig = { + hosts: ['http://8.8.8.8'], + logQueries: true, + ssl: { certificate: 'certificate-value' }, + }; + + prebootContract.createClient('some-custom-type', customConfig); + const config = MockClusterClient.mock.calls[0][0]; + + expect(config).toMatchInlineSnapshot(` + Object { + "healthCheckDelay": "PT0.01S", + "hosts": Array [ + "http://8.8.8.8", + ], + "logQueries": true, + "requestHeadersWhitelist": Array [ + undefined, + ], + "ssl": Object { + "certificate": "certificate-value", + "verificationMode": "none", + }, + } + `); + }); + }); +}); + describe('#setup', () => { it('returns legacy Elasticsearch config as a part of the contract', async () => { const setupContract = await elasticsearchService.setup(setupDeps); @@ -249,7 +333,7 @@ describe('#setup', () => { describe('#start', () => { it('throws if called before `setup`', async () => { - expect(() => elasticsearchService.start()).rejects.toMatchInlineSnapshot( + await expect(() => elasticsearchService.start()).rejects.toMatchInlineSnapshot( `[Error: ElasticsearchService needs to be setup before calling start]` ); }); diff --git a/src/core/server/elasticsearch/elasticsearch_service.ts b/src/core/server/elasticsearch/elasticsearch_service.ts index deb2d49f708176..f983a8b77fe085 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.ts @@ -18,11 +18,15 @@ import { ILegacyCustomClusterClient, LegacyElasticsearchClientConfig, } from './legacy'; -import { ClusterClient, ICustomClusterClient, ElasticsearchClientConfig } from './client'; +import { ClusterClient, ElasticsearchClientConfig } from './client'; import { ElasticsearchConfig, ElasticsearchConfigType } from './elasticsearch_config'; -import type { InternalHttpServiceSetup, GetAuthHeaders } from '../http/'; +import type { InternalHttpServiceSetup, GetAuthHeaders } from '../http'; import type { InternalExecutionContextSetup, IExecutionContext } from '../execution_context'; -import { InternalElasticsearchServiceSetup, InternalElasticsearchServiceStart } from './types'; +import { + InternalElasticsearchServicePreboot, + InternalElasticsearchServiceSetup, + InternalElasticsearchServiceStart, +} from './types'; import { pollEsNodesVersion } from './version_check/ensure_es_version'; import { calculateStatus$ } from './status'; @@ -57,6 +61,22 @@ export class ElasticsearchService .pipe(map((rawConfig) => new ElasticsearchConfig(rawConfig))); } + public async preboot(): Promise { + this.log.debug('Prebooting elasticsearch service'); + + const config = await this.config$.pipe(first()).toPromise(); + return { + config: { + hosts: config.hosts, + credentialsSpecified: + config.username !== undefined || + config.password !== undefined || + config.serviceAccountToken !== undefined, + }, + createClient: (type, clientConfig) => this.createClusterClient(type, config, clientConfig), + }; + } + public async setup(deps: SetupDeps): Promise { this.log.debug('Setting up elasticsearch service'); @@ -96,18 +116,9 @@ export class ElasticsearchService } const config = await this.config$.pipe(first()).toPromise(); - - const createClient = ( - type: string, - clientConfig: Partial = {} - ): ICustomClusterClient => { - const finalConfig = merge({}, config, clientConfig); - return this.createClusterClient(type, finalConfig); - }; - return { client: this.client!, - createClient, + createClient: (type, clientConfig) => this.createClusterClient(type, config, clientConfig), legacy: { config$: this.config$, client: this.legacyClient, @@ -127,7 +138,12 @@ export class ElasticsearchService } } - private createClusterClient(type: string, config: ElasticsearchClientConfig) { + private createClusterClient( + type: string, + baseConfig: ElasticsearchConfig, + clientConfig?: Partial + ) { + const config = clientConfig ? merge({}, baseConfig, clientConfig) : baseConfig; return new ClusterClient( config, this.coreContext.logger.get('elasticsearch'), diff --git a/src/core/server/elasticsearch/index.ts b/src/core/server/elasticsearch/index.ts index 94dc10ff4e8636..d97e3331c7cf5e 100644 --- a/src/core/server/elasticsearch/index.ts +++ b/src/core/server/elasticsearch/index.ts @@ -11,13 +11,16 @@ export { config, configSchema } from './elasticsearch_config'; export { ElasticsearchConfig } from './elasticsearch_config'; export type { NodesVersionCompatibility } from './version_check/ensure_es_version'; export type { + ElasticsearchServicePreboot, ElasticsearchServiceSetup, ElasticsearchServiceStart, ElasticsearchStatusMeta, + InternalElasticsearchServicePreboot, InternalElasticsearchServiceSetup, InternalElasticsearchServiceStart, FakeRequest, ScopeableRequest, + ElasticsearchConfigPreboot, } from './types'; export * from './legacy'; export type { diff --git a/src/core/server/elasticsearch/types.ts b/src/core/server/elasticsearch/types.ts index 8bbf665cbc0965..375c7015b16d79 100644 --- a/src/core/server/elasticsearch/types.ts +++ b/src/core/server/elasticsearch/types.ts @@ -19,6 +19,43 @@ import { IClusterClient, ICustomClusterClient, ElasticsearchClientConfig } from import { NodesVersionCompatibility } from './version_check/ensure_es_version'; import { ServiceStatus } from '../status'; +/** + * @public + */ +export interface ElasticsearchServicePreboot { + /** + * A limited set of Elasticsearch configuration entries. + * + * @example + * ```js + * const { hosts, credentialsSpecified } = core.elasticsearch.config; + * ``` + */ + readonly config: Readonly; + + /** + * Create application specific Elasticsearch cluster API client with customized config. See {@link IClusterClient}. + * + * @param type Unique identifier of the client + * @param clientConfig A config consists of Elasticsearch JS client options and + * valid sub-set of Elasticsearch service config. + * We fill all the missing properties in the `clientConfig` using the default + * Elasticsearch config so that we don't depend on default values set and + * controlled by underlying Elasticsearch JS client. + * We don't run validation against the passed config and expect it to be valid. + * + * @example + * ```js + * const client = elasticsearch.createClient('my-app-name', config); + * const data = await client.asInternalUser.search(); + * ``` + */ + readonly createClient: ( + type: string, + clientConfig?: Partial + ) => ICustomClusterClient; +} + /** * @public */ @@ -77,6 +114,9 @@ export interface ElasticsearchServiceSetup { }; } +/** @internal */ +export type InternalElasticsearchServicePreboot = ElasticsearchServicePreboot; + /** @internal */ export interface InternalElasticsearchServiceSetup extends ElasticsearchServiceSetup { esNodesCompatibility$: Observable; @@ -199,3 +239,21 @@ export interface FakeRequest { * See {@link KibanaRequest}. */ export type ScopeableRequest = KibanaRequest | LegacyRequest | FakeRequest; + +/** + * A limited set of Elasticsearch configuration entries exposed to the `preboot` plugins at `setup`. + * + * @public + */ +export interface ElasticsearchConfigPreboot { + /** + * Hosts that the client will connect to. If sniffing is enabled, this list will + * be used as seeds to discover the rest of your cluster. + */ + readonly hosts: string[]; + + /** + * Indicates whether Elasticsearch configuration includes credentials (`username`, `password` or `serviceAccountToken`). + */ + readonly credentialsSpecified: boolean; +} diff --git a/src/core/server/environment/environment_service.mock.ts b/src/core/server/environment/environment_service.mock.ts index 2bc5fa89b8e26d..ab620c790a157b 100644 --- a/src/core/server/environment/environment_service.mock.ts +++ b/src/core/server/environment/environment_service.mock.ts @@ -7,25 +7,39 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; -import type { EnvironmentService, InternalEnvironmentServiceSetup } from './environment_service'; +import type { + EnvironmentService, + InternalEnvironmentServicePreboot, + InternalEnvironmentServiceSetup, +} from './environment_service'; + +const createPrebootContractMock = () => { + const prebootContract: jest.Mocked = { + instanceUuid: 'uuid', + }; + return prebootContract; +}; const createSetupContractMock = () => { - const setupContract: jest.Mocked = { + const prebootContract: jest.Mocked = { instanceUuid: 'uuid', }; - return setupContract; + return prebootContract; }; type EnvironmentServiceContract = PublicMethodsOf; const createMock = () => { const mocked: jest.Mocked = { + preboot: jest.fn(), setup: jest.fn(), }; - mocked.setup.mockResolvedValue(createSetupContractMock()); + mocked.preboot.mockResolvedValue(createPrebootContractMock()); + mocked.setup.mockReturnValue(createSetupContractMock()); return mocked; }; export const environmentServiceMock = { create: createMock, + createPrebootContract: createPrebootContractMock, createSetupContract: createSetupContractMock, }; diff --git a/src/core/server/environment/environment_service.test.ts b/src/core/server/environment/environment_service.test.ts index fb3ddaa77b4162..34647d090b9957 100644 --- a/src/core/server/environment/environment_service.test.ts +++ b/src/core/server/environment/environment_service.test.ts @@ -76,9 +76,9 @@ describe('UuidService', () => { jest.clearAllMocks(); }); - describe('#setup()', () => { + describe('#preboot()', () => { it('calls resolveInstanceUuid with correct parameters', async () => { - await service.setup(); + await service.preboot(); expect(resolveInstanceUuid).toHaveBeenCalledTimes(1); expect(resolveInstanceUuid).toHaveBeenCalledWith({ @@ -89,7 +89,7 @@ describe('UuidService', () => { }); it('calls createDataFolder with correct parameters', async () => { - await service.setup(); + await service.preboot(); expect(createDataFolder).toHaveBeenCalledTimes(1); expect(createDataFolder).toHaveBeenCalledWith({ @@ -99,7 +99,7 @@ describe('UuidService', () => { }); it('calls writePidFile with correct parameters', async () => { - await service.setup(); + await service.preboot(); expect(writePidFile).toHaveBeenCalledTimes(1); expect(writePidFile).toHaveBeenCalledWith({ @@ -109,14 +109,14 @@ describe('UuidService', () => { }); it('returns the uuid resolved from resolveInstanceUuid', async () => { - const setup = await service.setup(); + const preboot = await service.preboot(); - expect(setup.instanceUuid).toEqual('SOME_UUID'); + expect(preboot.instanceUuid).toEqual('SOME_UUID'); }); describe('process warnings', () => { it('logs warnings coming from the process', async () => { - await service.setup(); + await service.preboot(); const warning = new Error('something went wrong'); process.emit('warning', warning); @@ -126,7 +126,7 @@ describe('UuidService', () => { }); it('does not log deprecation warnings', async () => { - await service.setup(); + await service.preboot(); const warning = new Error('something went wrong'); warning.name = 'DeprecationWarning'; @@ -136,4 +136,11 @@ describe('UuidService', () => { }); }); }); + + describe('#setup()', () => { + it('returns the uuid resolved from resolveInstanceUuid', async () => { + await expect(service.preboot()).resolves.toEqual({ instanceUuid: 'SOME_UUID' }); + expect(service.setup()).toEqual({ instanceUuid: 'SOME_UUID' }); + }); + }); }); diff --git a/src/core/server/environment/environment_service.ts b/src/core/server/environment/environment_service.ts index e652622049cfa1..f96b616256577a 100644 --- a/src/core/server/environment/environment_service.ts +++ b/src/core/server/environment/environment_service.ts @@ -20,13 +20,18 @@ import { writePidFile } from './write_pid_file'; /** * @internal */ -export interface InternalEnvironmentServiceSetup { +export interface InternalEnvironmentServicePreboot { /** * Retrieve the Kibana instance uuid. */ instanceUuid: string; } +/** + * @internal + */ +export type InternalEnvironmentServiceSetup = InternalEnvironmentServicePreboot; + /** @internal */ export class EnvironmentService { private readonly log: Logger; @@ -40,7 +45,9 @@ export class EnvironmentService { this.configService = core.configService; } - public async setup() { + public async preboot() { + // IMPORTANT: This code is based on the assumption that none of the configuration values used + // here is supposed to change during preboot phase and it's safe to read them only once. const [pathConfig, serverConfig, pidConfig] = await Promise.all([ this.configService.atPath(pathConfigDef.path).pipe(take(1)).toPromise(), this.configService.atPath(httpConfigDef.path).pipe(take(1)).toPromise(), @@ -73,4 +80,10 @@ export class EnvironmentService { instanceUuid: this.uuid, }; } + + public setup() { + return { + instanceUuid: this.uuid, + }; + } } diff --git a/src/core/server/environment/index.ts b/src/core/server/environment/index.ts index 01d5097887248c..886c0f667fdb4d 100644 --- a/src/core/server/environment/index.ts +++ b/src/core/server/environment/index.ts @@ -7,6 +7,9 @@ */ export { EnvironmentService } from './environment_service'; -export type { InternalEnvironmentServiceSetup } from './environment_service'; +export type { + InternalEnvironmentServicePreboot, + InternalEnvironmentServiceSetup, +} from './environment_service'; export { config } from './pid_config'; export type { PidConfigType } from './pid_config'; diff --git a/src/core/server/execution_context/integration_tests/tracing.test.ts b/src/core/server/execution_context/integration_tests/tracing.test.ts index 2c4057ddaf8b8a..ade67d0dd2605e 100644 --- a/src/core/server/execution_context/integration_tests/tracing.test.ts +++ b/src/core/server/execution_context/integration_tests/tracing.test.ts @@ -41,6 +41,7 @@ describe('trace', () => { }, }, }); + await root.preboot(); }, 30000); afterEach(async () => { @@ -167,6 +168,7 @@ describe('trace', () => { }, }, }); + await rootExecutionContextDisabled.preboot(); }, 30000); afterEach(async () => { diff --git a/src/core/server/http/__snapshots__/http_service.test.ts.snap b/src/core/server/http/__snapshots__/http_service.test.ts.snap index 04b78a84e818e4..0377525c741260 100644 --- a/src/core/server/http/__snapshots__/http_service.test.ts.snap +++ b/src/core/server/http/__snapshots__/http_service.test.ts.snap @@ -8,7 +8,7 @@ Array [ ] `; -exports[`spins up notReady server until started if configured with \`autoListen:true\`: 503 response 1`] = ` +exports[`spins up \`preboot\` server until started if configured with \`autoListen:true\`: 503 response 1`] = ` Object { "body": Array [ Array [ diff --git a/src/core/server/http/http_service.mock.ts b/src/core/server/http/http_service.mock.ts index a589bc76d21fcd..ef5e1510837808 100644 --- a/src/core/server/http/http_service.mock.ts +++ b/src/core/server/http/http_service.mock.ts @@ -12,6 +12,8 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { CspConfig } from '../csp'; import { mockRouter, RouterMock } from './router/router.mock'; import { + InternalHttpServicePreboot, + HttpServicePreboot, InternalHttpServiceSetup, HttpServiceSetup, HttpServiceStart, @@ -31,6 +33,10 @@ import { ExternalUrlConfig } from '../external_url'; type BasePathMocked = jest.Mocked; type AuthMocked = jest.Mocked; +export type HttpServicePrebootMock = jest.Mocked; +export type InternalHttpServicePrebootMock = jest.Mocked< + Omit +> & { basePath: BasePathMocked }; export type HttpServiceSetupMock = jest.Mocked< Omit > & { @@ -72,6 +78,31 @@ const createAuthMock = () => { return mock; }; +const createInternalPrebootContractMock = () => { + const mock: InternalHttpServicePrebootMock = { + registerRoutes: jest.fn(), + // @ts-expect-error tsc cannot infer ContextName and uses never + registerRouteHandlerContext: jest.fn(), + registerStaticDir: jest.fn(), + basePath: createBasePathMock(), + csp: CspConfig.DEFAULT, + externalUrl: ExternalUrlConfig.DEFAULT, + auth: createAuthMock(), + }; + return mock; +}; + +const createPrebootContractMock = () => { + const internalMock = createInternalPrebootContractMock(); + + const mock: HttpServicePrebootMock = { + registerRoutes: internalMock.registerRoutes, + basePath: createBasePathMock(), + }; + + return mock; +}; + const createInternalSetupContractMock = () => { const mock: InternalHttpServiceSetupMock = { // we can mock other hapi server methods when we need it @@ -100,6 +131,7 @@ const createInternalSetupContractMock = () => { auth: createAuthMock(), getAuthHeaders: jest.fn(), getServerInfo: jest.fn(), + registerPrebootRoutes: jest.fn(), }; mock.createCookieSessionStorageFactory.mockResolvedValue(sessionStorageMock.createFactory()); mock.createRouter.mockImplementation(() => mockRouter.create()); @@ -165,11 +197,13 @@ type HttpServiceContract = PublicMethodsOf; const createHttpServiceMock = () => { const mocked: jest.Mocked = { + preboot: jest.fn(), setup: jest.fn(), getStartContract: jest.fn(), start: jest.fn(), stop: jest.fn(), }; + mocked.preboot.mockResolvedValue(createInternalPrebootContractMock()); mocked.setup.mockResolvedValue(createInternalSetupContractMock()); mocked.getStartContract.mockReturnValue(createInternalStartContractMock()); mocked.start.mockResolvedValue(createInternalStartContractMock()); @@ -204,6 +238,8 @@ export const httpServiceMock = { create: createHttpServiceMock, createBasePath: createBasePathMock, createAuth: createAuthMock, + createInternalPrebootContract: createInternalPrebootContractMock, + createPrebootContract: createPrebootContractMock, createInternalSetupContract: createInternalSetupContractMock, createSetupContract: createSetupContractMock, createInternalStartContract: createInternalStartContractMock, diff --git a/src/core/server/http/http_service.test.ts b/src/core/server/http/http_service.test.ts index d8a7b542754801..8d29e3221a2ca3 100644 --- a/src/core/server/http/http_service.test.ts +++ b/src/core/server/http/http_service.test.ts @@ -20,7 +20,8 @@ import { loggingSystemMock } from '../logging/logging_system.mock'; import { contextServiceMock } from '../context/context_service.mock'; import { executionContextServiceMock } from '../execution_context/execution_context_service.mock'; import { config as cspConfig } from '../csp'; -import { config as externalUrlConfig } from '../external_url'; +import { config as externalUrlConfig, ExternalUrlConfig } from '../external_url'; +import { Router } from './router'; const logger = loggingSystemMock.create(); const env = Env.createDefault(REPO_ROOT, getEnvOptions()); @@ -42,8 +43,12 @@ const createConfigService = (value: Partial = {}) => { configService.setSchema(externalUrlConfig.path, externalUrlConfig.schema); return configService; }; +const contextPreboot = contextServiceMock.createPrebootContract(); const contextSetup = contextServiceMock.createSetupContract(); +const prebootDeps = { + context: contextPreboot, +}; const setupDeps = { context: contextSetup, executionContext: executionContextServiceMock.createInternalSetupContract(), @@ -70,35 +75,40 @@ test('creates and sets up http server', async () => { start: jest.fn(), stop: jest.fn(), }; - const notReadyHttpServer = { + const prebootHttpServer = { isListening: () => false, - setup: jest.fn().mockReturnValue({ server: fakeHapiServer }), + setup: jest.fn().mockReturnValue({ server: fakeHapiServer, registerStaticDir: jest.fn() }), start: jest.fn(), stop: jest.fn(), }; + mockHttpServer.mockImplementationOnce(() => prebootHttpServer); mockHttpServer.mockImplementationOnce(() => httpServer); - mockHttpServer.mockImplementationOnce(() => notReadyHttpServer); const service = new HttpService({ coreId, configService, env, logger }); - expect(mockHttpServer.mock.instances.length).toBe(1); + expect(mockHttpServer.mock.instances.length).toBe(2); expect(httpServer.setup).not.toHaveBeenCalled(); - expect(notReadyHttpServer.setup).not.toHaveBeenCalled(); + expect(prebootHttpServer.setup).not.toHaveBeenCalled(); + + await service.preboot(prebootDeps); + expect(httpServer.setup).not.toHaveBeenCalled(); + expect(httpServer.start).not.toHaveBeenCalled(); + + expect(prebootHttpServer.setup).toHaveBeenCalled(); + expect(prebootHttpServer.start).toHaveBeenCalled(); await service.setup(setupDeps); expect(httpServer.setup).toHaveBeenCalled(); expect(httpServer.start).not.toHaveBeenCalled(); - - expect(notReadyHttpServer.setup).toHaveBeenCalled(); - expect(notReadyHttpServer.start).toHaveBeenCalled(); + expect(prebootHttpServer.stop).not.toHaveBeenCalled(); await service.start(); expect(httpServer.start).toHaveBeenCalled(); - expect(notReadyHttpServer.stop).toHaveBeenCalled(); + expect(prebootHttpServer.stop).toHaveBeenCalled(); }); -test('spins up notReady server until started if configured with `autoListen:true`', async () => { +test('spins up `preboot` server until started if configured with `autoListen:true`', async () => { const configService = createConfigService(); const httpServer = { isListening: () => false, @@ -106,19 +116,19 @@ test('spins up notReady server until started if configured with `autoListen:true start: jest.fn(), stop: jest.fn(), }; - const notReadyHapiServer = { + const prebootHapiServer = { start: jest.fn(), stop: jest.fn(), route: jest.fn(), }; mockHttpServer - .mockImplementationOnce(() => httpServer) .mockImplementationOnce(() => ({ - setup: () => ({ server: notReadyHapiServer }), + setup: () => ({ server: prebootHapiServer, registerStaticDir: jest.fn() }), start: jest.fn(), - stop: jest.fn().mockImplementation(() => notReadyHapiServer.stop()), - })); + stop: jest.fn().mockImplementation(() => prebootHapiServer.stop()), + })) + .mockImplementationOnce(() => httpServer); const service = new HttpService({ coreId, @@ -127,7 +137,7 @@ test('spins up notReady server until started if configured with `autoListen:true logger, }); - await service.setup(setupDeps); + await service.preboot(prebootDeps); const mockResponse: any = { code: jest.fn().mockImplementation(() => mockResponse), @@ -137,7 +147,7 @@ test('spins up notReady server until started if configured with `autoListen:true response: jest.fn().mockReturnValue(mockResponse), }; - const [[{ handler }]] = notReadyHapiServer.route.mock.calls; + const [[{ handler }]] = prebootHapiServer.route.mock.calls; const response503 = await handler(httpServerMock.createRawRequest(), mockResponseToolkit); expect(response503).toBe(mockResponse); expect({ @@ -146,15 +156,25 @@ test('spins up notReady server until started if configured with `autoListen:true header: mockResponse.header.mock.calls, }).toMatchSnapshot('503 response'); + await service.setup(setupDeps); await service.start(); expect(httpServer.start).toBeCalledTimes(1); - expect(notReadyHapiServer.stop).toBeCalledTimes(1); + expect(prebootHapiServer.stop).toBeCalledTimes(1); }); test('logs error if already set up', async () => { const configService = createConfigService(); + mockHttpServer.mockImplementationOnce(() => ({ + setup: () => ({ + server: { start: jest.fn(), stop: jest.fn(), route: jest.fn() }, + registerStaticDir: jest.fn(), + }), + start: noop, + stop: noop, + })); + const httpServer = { isListening: () => true, setup: jest.fn().mockReturnValue({ server: fakeHapiServer }), @@ -165,6 +185,7 @@ test('logs error if already set up', async () => { const service = new HttpService({ coreId, configService, env, logger }); + await service.preboot(prebootDeps); await service.setup(setupDeps); expect(loggingSystemMock.collect(logger).warn).toMatchSnapshot(); @@ -179,29 +200,30 @@ test('stops http server', async () => { start: noop, stop: jest.fn(), }; - const notReadyHttpServer = { + const prebootHttpServer = { isListening: () => false, - setup: jest.fn().mockReturnValue({ server: fakeHapiServer }), + setup: jest.fn().mockReturnValue({ server: fakeHapiServer, registerStaticDir: jest.fn() }), start: noop, stop: jest.fn(), }; + mockHttpServer.mockImplementationOnce(() => prebootHttpServer); mockHttpServer.mockImplementationOnce(() => httpServer); - mockHttpServer.mockImplementationOnce(() => notReadyHttpServer); const service = new HttpService({ coreId, configService, env, logger }); + await service.preboot(prebootDeps); await service.setup(setupDeps); await service.start(); expect(httpServer.stop).toHaveBeenCalledTimes(0); - expect(notReadyHttpServer.stop).toHaveBeenCalledTimes(1); + expect(prebootHttpServer.stop).toHaveBeenCalledTimes(1); await service.stop(); expect(httpServer.stop).toHaveBeenCalledTimes(1); }); -test('stops not ready server if it is running', async () => { +test('stops `preboot` server if it is running', async () => { const configService = createConfigService(); const mockHapiServer = { start: jest.fn(), @@ -210,7 +232,7 @@ test('stops not ready server if it is running', async () => { }; const httpServer = { isListening: () => false, - setup: jest.fn().mockReturnValue({ server: mockHapiServer }), + setup: jest.fn().mockReturnValue({ server: mockHapiServer, registerStaticDir: jest.fn() }), start: noop, stop: jest.fn().mockImplementation(() => mockHapiServer.stop()), }; @@ -218,16 +240,61 @@ test('stops not ready server if it is running', async () => { const service = new HttpService({ coreId, configService, env, logger }); - await service.setup(setupDeps); + await service.preboot(prebootDeps); await service.stop(); expect(mockHapiServer.stop).toHaveBeenCalledTimes(2); }); +test('does not try to stop `preboot` server if it has been already stopped', async () => { + const prebootHttpServer = { + isListening: () => false, + setup: jest.fn().mockReturnValue({ server: fakeHapiServer, registerStaticDir: jest.fn() }), + start: noop, + stop: jest.fn(), + }; + const standardHttpServer = { + isListening: () => false, + setup: jest.fn().mockReturnValue({ server: fakeHapiServer }), + start: noop, + stop: jest.fn(), + }; + + mockHttpServer + .mockImplementationOnce(() => prebootHttpServer) + .mockImplementationOnce(() => standardHttpServer); + + const service = new HttpService({ coreId, configService: createConfigService(), env, logger }); + await service.preboot(prebootDeps); + await service.setup(setupDeps); + + expect(prebootHttpServer.stop).not.toHaveBeenCalled(); + expect(standardHttpServer.stop).not.toHaveBeenCalled(); + + await service.start(); + + expect(prebootHttpServer.stop).toHaveBeenCalledTimes(1); + expect(standardHttpServer.stop).not.toHaveBeenCalled(); + + await service.stop(); + + expect(prebootHttpServer.stop).toHaveBeenCalledTimes(1); + expect(standardHttpServer.stop).toHaveBeenCalledTimes(1); +}); + test('register route handler', async () => { const configService = createConfigService(); + mockHttpServer.mockImplementationOnce(() => ({ + setup: () => ({ + server: { start: jest.fn(), stop: jest.fn(), route: jest.fn() }, + registerStaticDir: jest.fn(), + }), + start: noop, + stop: noop, + })); + const registerRouterMock = jest.fn(); const httpServer = { isListening: () => false, @@ -241,6 +308,7 @@ test('register route handler', async () => { const service = new HttpService({ coreId, configService, env, logger }); + await service.preboot(prebootDeps); const { createRouter } = await service.setup(setupDeps); const router = createRouter('/foo'); @@ -248,10 +316,103 @@ test('register route handler', async () => { expect(registerRouterMock).toHaveBeenLastCalledWith(router); }); +test('register preboot route handler on preboot', async () => { + const registerRouterMock = jest.fn(); + mockHttpServer.mockImplementationOnce(() => ({ + setup: () => ({ + server: { start: jest.fn(), stop: jest.fn(), route: jest.fn() }, + registerStaticDir: jest.fn(), + registerRouterAfterListening: registerRouterMock, + }), + start: noop, + stop: noop, + })); + + const service = new HttpService({ coreId, configService: createConfigService(), env, logger }); + + const registerRoutesMock = jest.fn(); + const { registerRoutes } = await service.preboot(prebootDeps); + registerRoutes('some-path', registerRoutesMock); + + expect(registerRoutesMock).toHaveBeenCalledTimes(1); + expect(registerRoutesMock).toHaveBeenCalledWith(expect.any(Router)); + + const [[router]] = registerRoutesMock.mock.calls; + expect(registerRouterMock).toHaveBeenCalledTimes(1); + expect(registerRouterMock).toHaveBeenCalledWith(router); +}); + +test('register preboot route handler on setup', async () => { + const registerRouterMock = jest.fn(); + mockHttpServer + .mockImplementationOnce(() => ({ + setup: () => ({ + server: { start: jest.fn(), stop: jest.fn(), route: jest.fn() }, + registerStaticDir: jest.fn(), + registerRouterAfterListening: registerRouterMock, + }), + start: noop, + stop: noop, + })) + .mockImplementationOnce(() => ({ setup: () => ({ server: {} }), start: noop, stop: noop })); + + const service = new HttpService({ coreId, configService: createConfigService(), env, logger }); + await service.preboot(prebootDeps); + + const registerRoutesMock = jest.fn(); + const { registerPrebootRoutes } = await service.setup(setupDeps); + registerPrebootRoutes('some-path', registerRoutesMock); + + expect(registerRoutesMock).toHaveBeenCalledTimes(1); + expect(registerRoutesMock).toHaveBeenCalledWith(expect.any(Router)); + + const [[router]] = registerRoutesMock.mock.calls; + expect(registerRouterMock).toHaveBeenCalledTimes(1); + expect(registerRouterMock).toHaveBeenCalledWith(router); +}); + +test('returns `preboot` http server contract on preboot', async () => { + const configService = createConfigService(); + const httpServer = { + server: fakeHapiServer, + registerStaticDir: jest.fn(), + auth: Symbol('auth'), + basePath: Symbol('basePath'), + csp: Symbol('csp'), + }; + + mockHttpServer.mockImplementation(() => ({ + isListening: () => false, + setup: jest.fn().mockReturnValue(httpServer), + start: noop, + stop: noop, + })); + + const service = new HttpService({ coreId, configService, env, logger }); + await expect(service.preboot(prebootDeps)).resolves.toMatchObject({ + auth: httpServer.auth, + basePath: httpServer.basePath, + csp: httpServer.csp, + externalUrl: expect.any(ExternalUrlConfig), + registerRouteHandlerContext: expect.any(Function), + registerRoutes: expect.any(Function), + registerStaticDir: expect.any(Function), + }); +}); + test('returns http server contract on setup', async () => { const configService = createConfigService(); const httpServer = { server: fakeHapiServer, options: { someOption: true } }; + mockHttpServer.mockImplementationOnce(() => ({ + setup: () => ({ + server: { start: jest.fn(), stop: jest.fn(), route: jest.fn() }, + registerStaticDir: jest.fn(), + }), + start: noop, + stop: noop, + })); + mockHttpServer.mockImplementation(() => ({ isListening: () => false, setup: jest.fn().mockReturnValue(httpServer), @@ -260,10 +421,12 @@ test('returns http server contract on setup', async () => { })); const service = new HttpService({ coreId, configService, env, logger }); + await service.preboot(prebootDeps); const setupContract = await service.setup(setupDeps); expect(setupContract).toMatchObject(httpServer); expect(setupContract).toMatchObject({ createRouter: expect.any(Function), + registerPrebootRoutes: expect.any(Function), }); }); @@ -271,6 +434,14 @@ test('does not start http server if configured with `autoListen:false`', async ( const configService = createConfigService({ autoListen: false, }); + mockHttpServer.mockImplementationOnce(() => ({ + setup: () => ({ + server: { start: jest.fn(), stop: jest.fn(), route: jest.fn() }, + registerStaticDir: jest.fn(), + }), + start: noop, + stop: noop, + })); const httpServer = { isListening: () => false, setup: jest.fn().mockReturnValue({}), @@ -286,6 +457,7 @@ test('does not start http server if configured with `autoListen:false`', async ( logger, }); + await service.preboot(prebootDeps); await service.setup(setupDeps); await service.start(); diff --git a/src/core/server/http/http_service.ts b/src/core/server/http/http_service.ts index 0097aab82b21c2..4b9e45e271be2b 100644 --- a/src/core/server/http/http_service.ts +++ b/src/core/server/http/http_service.ts @@ -6,21 +6,21 @@ * Side Public License, v 1. */ -import { Observable, Subscription, combineLatest, of } from 'rxjs'; +import { Observable, Subscription, combineLatest } from 'rxjs'; import { first, map } from 'rxjs/operators'; import { pick } from '@kbn/std'; import type { RequestHandlerContext } from 'src/core/server'; import type { InternalExecutionContextSetup } from '../execution_context'; import { CoreService } from '../../types'; -import { Logger, LoggerFactory } from '../logging'; -import { ContextSetup } from '../context'; +import { Logger } from '../logging'; +import { ContextSetup, InternalContextPreboot } from '../context'; import { Env } from '../config'; import { CoreContext } from '../core_context'; import { PluginOpaqueId } from '../plugins'; import { CspConfigType, config as cspConfig } from '../csp'; -import { IRouter, Router } from './router'; +import { Router } from './router'; import { HttpConfig, HttpConfigType, config as httpConfig } from './http_config'; import { HttpServer } from './http_server'; import { HttpsRedirectServer } from './https_redirect_server'; @@ -28,9 +28,9 @@ import { HttpsRedirectServer } from './https_redirect_server'; import { RequestHandlerContextContainer, RequestHandlerContextProvider, + InternalHttpServicePreboot, InternalHttpServiceSetup, InternalHttpServiceStart, - InternalNotReadyHttpServiceSetup, } from './types'; import { registerCoreHandlers } from './lifecycle_handlers'; @@ -40,6 +40,10 @@ import { ExternalUrlConfig, } from '../external_url'; +interface PrebootDeps { + context: InternalContextPreboot; +} + interface SetupDeps { context: ContextSetup; executionContext: InternalExecutionContextSetup; @@ -48,22 +52,22 @@ interface SetupDeps { /** @internal */ export class HttpService implements CoreService { + private readonly prebootServer: HttpServer; + private isPrebootServerStopped = false; private readonly httpServer: HttpServer; private readonly httpsRedirectServer: HttpsRedirectServer; private readonly config$: Observable; private configSubscription?: Subscription; - private readonly logger: LoggerFactory; private readonly log: Logger; private readonly env: Env; - private notReadyServer?: HttpServer; + private internalPreboot?: InternalHttpServicePreboot; private internalSetup?: InternalHttpServiceSetup; private requestHandlerContext?: RequestHandlerContextContainer; constructor(private readonly coreContext: CoreContext) { const { logger, configService, env } = coreContext; - this.logger = logger; this.env = env; this.log = logger.get('http'); this.config$ = combineLatest([ @@ -72,10 +76,63 @@ export class HttpService configService.atPath(externalUrlConfig.path), ]).pipe(map(([http, csp, externalUrl]) => new HttpConfig(http, csp, externalUrl))); const shutdownTimeout$ = this.config$.pipe(map(({ shutdownTimeout }) => shutdownTimeout)); + this.prebootServer = new HttpServer(logger, 'Preboot', shutdownTimeout$); this.httpServer = new HttpServer(logger, 'Kibana', shutdownTimeout$); this.httpsRedirectServer = new HttpsRedirectServer(logger.get('http', 'redirect', 'server')); } + public async preboot(deps: PrebootDeps): Promise { + this.log.debug('setting up preboot server'); + const config = await this.config$.pipe(first()).toPromise(); + + const prebootSetup = await this.prebootServer.setup(config); + prebootSetup.server.route({ + path: '/{p*}', + method: '*', + handler: (req, responseToolkit) => { + this.log.debug(`Kibana server is not ready yet ${req.method}:${req.url.href}.`); + + // If server is not ready yet, because plugins or core can perform + // long running tasks (build assets, saved objects migrations etc.) + // we should let client know that and ask to retry after 30 seconds. + return responseToolkit + .response('Kibana server is not ready yet') + .code(503) + .header('Retry-After', '30'); + }, + }); + + if (this.shouldListen(config)) { + this.log.debug('starting preboot server'); + await this.prebootServer.start(); + } + + const prebootServerRequestHandlerContext = deps.context.createContextContainer(); + this.internalPreboot = { + externalUrl: new ExternalUrlConfig(config.externalUrl), + csp: prebootSetup.csp, + basePath: prebootSetup.basePath, + registerStaticDir: prebootSetup.registerStaticDir.bind(prebootSetup), + auth: prebootSetup.auth, + server: prebootSetup.server, + registerRouteHandlerContext: (pluginOpaqueId, contextName, provider) => + prebootServerRequestHandlerContext.registerContext(pluginOpaqueId, contextName, provider), + registerRoutes: (path, registerCallback) => { + const router = new Router( + path, + this.log, + prebootServerRequestHandlerContext.createHandler.bind(null, this.coreContext.coreId) + ); + + registerCallback(router); + + prebootSetup.registerRouterAfterListening(router); + }, + }; + + return this.internalPreboot; + } + public async setup(deps: SetupDeps) { this.requestHandlerContext = deps.context.createContextContainer(); this.configSubscription = this.config$.subscribe(() => { @@ -90,8 +147,6 @@ export class HttpService const config = await this.config$.pipe(first()).toPromise(); - const notReadyServer = await this.setupNotReadyService({ config, context: deps.context }); - const { registerRouter, ...serverContract } = await this.httpServer.setup( config, deps.executionContext @@ -102,8 +157,6 @@ export class HttpService this.internalSetup = { ...serverContract, - notReadyServer, - externalUrl: new ExternalUrlConfig(config.externalUrl), createRouter: ( @@ -124,6 +177,8 @@ export class HttpService contextName: ContextName, provider: RequestHandlerContextProvider ) => this.requestHandlerContext!.registerContext(pluginOpaqueId, contextName, provider), + + registerPrebootRoutes: this.internalPreboot!.registerRoutes, }; return this.internalSetup; @@ -141,11 +196,10 @@ export class HttpService public async start() { const config = await this.config$.pipe(first()).toPromise(); if (this.shouldListen(config)) { - if (this.notReadyServer) { - this.log.debug('stopping NotReady server'); - await this.notReadyServer.stop(); - this.notReadyServer = undefined; - } + this.log.debug('stopping preboot server'); + await this.prebootServer.stop(); + this.isPrebootServerStopped = true; + // If a redirect port is specified, we start an HTTP server at this port and // redirect all requests to the SSL port. if (config.ssl.enabled && config.ssl.redirectHttpFromPort !== undefined) { @@ -169,81 +223,15 @@ export class HttpService } public async stop() { - if (this.configSubscription === undefined) { - return; - } - this.configSubscription?.unsubscribe(); this.configSubscription = undefined; - if (this.notReadyServer) { - await this.notReadyServer.stop(); + if (!this.isPrebootServerStopped) { + this.isPrebootServerStopped = false; + await this.prebootServer.stop(); } + await this.httpServer.stop(); await this.httpsRedirectServer.stop(); } - - private async setupNotReadyService({ - config, - context, - }: { - config: HttpConfig; - context: ContextSetup; - }): Promise { - if (!this.shouldListen(config)) { - return; - } - - const notReadySetup = await this.runNotReadyServer(config); - - // We cannot use the real context container since the core services may not yet be ready - const fakeContext: RequestHandlerContextContainer = new Proxy( - context.createContextContainer(), - { - get: (target, property, receiver) => { - if (property === 'createHandler') { - return Reflect.get(target, property, receiver); - } - throw new Error(`Unexpected access from fake context: ${String(property)}`); - }, - } - ); - - return { - registerRoutes: (path: string, registerCallback: (router: IRouter) => void) => { - const router = new Router( - path, - this.log, - fakeContext.createHandler.bind(null, this.coreContext.coreId) - ); - - registerCallback(router); - notReadySetup.registerRouterAfterListening(router); - }, - }; - } - - private async runNotReadyServer(config: HttpConfig) { - this.log.debug('starting NotReady server'); - this.notReadyServer = new HttpServer(this.logger, 'NotReady', of(config.shutdownTimeout)); - const notReadySetup = await this.notReadyServer.setup(config); - notReadySetup.server.route({ - path: '/{p*}', - method: '*', - handler: (req, responseToolkit) => { - this.log.debug(`Kibana server is not ready yet ${req.method}:${req.url.href}.`); - - // If server is not ready yet, because plugins or core can perform - // long running tasks (build assets, saved objects migrations etc.) - // we should let client know that and ask to retry after 30 seconds. - return responseToolkit - .response('Kibana server is not ready yet') - .code(503) - .header('Retry-After', '30'); - }, - }); - await this.notReadyServer.start(); - - return notReadySetup; - } } diff --git a/src/core/server/http/index.ts b/src/core/server/http/index.ts index 84fe5149c89c66..cad5a50dbc5050 100644 --- a/src/core/server/http/index.ts +++ b/src/core/server/http/index.ts @@ -87,6 +87,8 @@ export type { RequestHandlerContextContainer, RequestHandlerContextProvider, HttpAuth, + HttpServicePreboot, + InternalHttpServicePreboot, HttpServiceSetup, InternalHttpServiceSetup, HttpServiceStart, diff --git a/src/core/server/http/integration_tests/core_services.test.ts b/src/core/server/http/integration_tests/core_services.test.ts index 99b63fc73687a1..e497f254e06329 100644 --- a/src/core/server/http/integration_tests/core_services.test.ts +++ b/src/core/server/http/integration_tests/core_services.test.ts @@ -43,6 +43,7 @@ describe('http service', () => { let root: ReturnType; beforeEach(async () => { root = kbnTestServer.createRoot({ plugins: { initialize: false } }); + await root.preboot(); }, 30000); afterEach(async () => { @@ -189,6 +190,7 @@ describe('http service', () => { let root: ReturnType; beforeEach(async () => { root = kbnTestServer.createRoot({ plugins: { initialize: false } }); + await root.preboot(); }, 30000); afterEach(async () => { @@ -282,6 +284,7 @@ describe('http service', () => { beforeEach(async () => { root = kbnTestServer.createRoot({ plugins: { initialize: false } }); + await root.preboot(); }, 30000); afterEach(async () => { diff --git a/src/core/server/http/integration_tests/http_auth.test.ts b/src/core/server/http/integration_tests/http_auth.test.ts index 0696deb9c07ae4..9c923943118a09 100644 --- a/src/core/server/http/integration_tests/http_auth.test.ts +++ b/src/core/server/http/integration_tests/http_auth.test.ts @@ -15,6 +15,7 @@ describe('http auth', () => { beforeEach(async () => { root = kbnTestServer.createRoot({ plugins: { initialize: false } }); + await root.preboot(); }, 30000); afterEach(async () => { diff --git a/src/core/server/http/integration_tests/lifecycle.test.ts b/src/core/server/http/integration_tests/lifecycle.test.ts index da8abe55b65922..e883cd59c8c77b 100644 --- a/src/core/server/http/integration_tests/lifecycle.test.ts +++ b/src/core/server/http/integration_tests/lifecycle.test.ts @@ -29,9 +29,10 @@ const setupDeps = { executionContext: executionContextServiceMock.createInternalSetupContract(), }; -beforeEach(() => { +beforeEach(async () => { logger = loggingSystemMock.create(); server = createHttpServer({ logger }); + await server.preboot({ context: contextServiceMock.createPrebootContract() }); }); afterEach(async () => { diff --git a/src/core/server/http/integration_tests/lifecycle_handlers.test.ts b/src/core/server/http/integration_tests/lifecycle_handlers.test.ts index 077e2f6e9c4850..c633db11edd7a4 100644 --- a/src/core/server/http/integration_tests/lifecycle_handlers.test.ts +++ b/src/core/server/http/integration_tests/lifecycle_handlers.test.ts @@ -91,6 +91,7 @@ describe('core lifecycle handlers', () => { }); server = createHttpServer({ configService }); + await server.preboot({ context: contextServiceMock.createPrebootContract() }); const serverSetup = await server.setup(setupDeps); router = serverSetup.createRouter('/'); innerServer = serverSetup.server; diff --git a/src/core/server/http/integration_tests/logging.test.ts b/src/core/server/http/integration_tests/logging.test.ts index 62cb699bc49f6b..f7eee9580d11a7 100644 --- a/src/core/server/http/integration_tests/logging.test.ts +++ b/src/core/server/http/integration_tests/logging.test.ts @@ -28,6 +28,7 @@ describe('request logging', () => { describe('configuration', () => { it('does not log with a default config', async () => { const root = kbnTestServer.createRoot({ plugins: { initialize: false } }); + await root.preboot(); const { http } = await root.setup(); http @@ -69,6 +70,7 @@ describe('request logging', () => { initialize: false, }, }); + await root.preboot(); const { http } = await root.setup(); http @@ -125,6 +127,7 @@ describe('request logging', () => { }); it('handles a GET request', async () => { + await root.preboot(); const { http } = await root.setup(); http @@ -147,6 +150,7 @@ describe('request logging', () => { }); it('handles a POST request', async () => { + await root.preboot(); const { http } = await root.setup(); http.createRouter('/').post( @@ -178,6 +182,7 @@ describe('request logging', () => { }); it('handles an error response', async () => { + await root.preboot(); const { http } = await root.setup(); http @@ -198,6 +203,7 @@ describe('request logging', () => { }); it('handles query strings', async () => { + await root.preboot(); const { http } = await root.setup(); http @@ -216,6 +222,7 @@ describe('request logging', () => { }); it('correctly calculates response payload', async () => { + await root.preboot(); const { http } = await root.setup(); http @@ -234,6 +241,7 @@ describe('request logging', () => { describe('handles request/response headers', () => { it('includes request/response headers in log entry', async () => { + await root.preboot(); const { http } = await root.setup(); http @@ -252,6 +260,7 @@ describe('request logging', () => { }); it('filters sensitive request headers by default', async () => { + await root.preboot(); const { http } = await root.setup(); http.createRouter('/').post( @@ -319,6 +328,7 @@ describe('request logging', () => { initialize: false, }, }); + await root.preboot(); const { http } = await root.setup(); http.createRouter('/').post( @@ -351,6 +361,7 @@ describe('request logging', () => { }); it('filters sensitive response headers by defaut', async () => { + await root.preboot(); const { http } = await root.setup(); http.createRouter('/').post( @@ -416,6 +427,7 @@ describe('request logging', () => { initialize: false, }, }); + await root.preboot(); const { http } = await root.setup(); http.createRouter('/').post( @@ -449,6 +461,7 @@ describe('request logging', () => { }); it('handles user agent', async () => { + await root.preboot(); const { http } = await root.setup(); http diff --git a/src/core/server/http/integration_tests/preboot.test.ts b/src/core/server/http/integration_tests/preboot.test.ts new file mode 100644 index 00000000000000..7c2118bea725b0 --- /dev/null +++ b/src/core/server/http/integration_tests/preboot.test.ts @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import supertest from 'supertest'; + +import { contextServiceMock } from '../../context/context_service.mock'; +import { executionContextServiceMock } from '../../execution_context/execution_context_service.mock'; +import { loggingSystemMock } from '../../logging/logging_system.mock'; +import { createHttpServer } from '../test_utils'; +import { HttpService } from '../http_service'; + +let server: HttpService; +const prebootDeps = { + context: contextServiceMock.createPrebootContract(), +}; +const setupDeps = { + context: contextServiceMock.createSetupContract(), + executionContext: executionContextServiceMock.createInternalSetupContract(), +}; + +beforeEach(async () => { + server = createHttpServer({ logger: loggingSystemMock.create() }); +}); + +afterEach(async () => { + await server.stop(); +}); + +describe('Preboot HTTP server', () => { + it('accepts requests before `setup`', async () => { + const { server: innerPrebootServer, registerRoutes } = await server.preboot(prebootDeps); + registerRoutes('', (router) => { + router.get({ path: '/preboot-get', validate: false }, (context, req, res) => + res.ok({ body: 'hello-get' }) + ); + router.post({ path: '/preboot-post', validate: false }, (context, req, res) => + res.ok({ body: 'hello-post' }) + ); + }); + + // Preboot routes should work now. + await supertest(innerPrebootServer.listener).get('/preboot-get').expect(200, 'hello-get'); + await supertest(innerPrebootServer.listener).post('/preboot-post').expect(200, 'hello-post'); + + // All non-preboot routes should get `503` (e.g. if client tries to access any standard API). + await supertest(innerPrebootServer.listener) + .get('/standard-get') + .expect(503, 'Kibana server is not ready yet'); + await supertest(innerPrebootServer.listener) + .post('/standard-post') + .expect(503, 'Kibana server is not ready yet'); + }); + + it('accepts requests after `setup`, but before `start`', async () => { + const { server: innerPrebootServer, registerRoutes } = await server.preboot(prebootDeps); + registerRoutes('', (router) => { + router.get({ path: '/preboot-get', validate: false }, (context, req, res) => + res.ok({ body: 'hello-get' }) + ); + router.post({ path: '/preboot-post', validate: false }, (context, req, res) => + res.ok({ body: 'hello-post' }) + ); + }); + + const { createRouter, server: innerStandardServer } = await server.setup(setupDeps); + const standardRouter = createRouter(''); + standardRouter.get({ path: '/standard-get', validate: false }, (context, req, res) => + res.ok({ body: 'hello-get' }) + ); + standardRouter.post({ path: '/standard-post', validate: false }, (context, req, res) => + res.ok({ body: 'hello-post' }) + ); + + // Preboot routes should still work. + await supertest(innerPrebootServer.listener).get('/preboot-get').expect(200, 'hello-get'); + await supertest(innerPrebootServer.listener).post('/preboot-post').expect(200, 'hello-post'); + + // All non-preboot routes should still get `503` (e.g. if client tries to access any standard API). + await supertest(innerPrebootServer.listener) + .get('/standard-get') + .expect(503, 'Kibana server is not ready yet'); + await supertest(innerPrebootServer.listener) + .post('/standard-post') + .expect(503, 'Kibana server is not ready yet'); + + // Standard HTTP server isn't functional yet. + await supertest(innerStandardServer.listener) + .get('/standard-get') + .expect(404, { statusCode: 404, error: 'Not Found', message: 'Not Found' }); + await supertest(innerStandardServer.listener) + .post('/standard-post') + .expect(404, { statusCode: 404, error: 'Not Found', message: 'Not Found' }); + }); + + it('is not available after `start`', async () => { + const { server: innerPrebootServer, registerRoutes } = await server.preboot(prebootDeps); + registerRoutes('', (router) => { + router.get({ path: '/preboot-get', validate: false }, (context, req, res) => + res.ok({ body: 'hello-get' }) + ); + router.post({ path: '/preboot-post', validate: false }, (context, req, res) => + res.ok({ body: 'hello-post' }) + ); + }); + + const { createRouter, server: innerStandardServer } = await server.setup(setupDeps); + const standardRouter = createRouter(''); + standardRouter.get({ path: '/standard-get', validate: false }, (context, req, res) => + res.ok({ body: 'hello-get' }) + ); + standardRouter.post({ path: '/standard-post', validate: false }, (context, req, res) => + res.ok({ body: 'hello-post' }) + ); + + await server.start(); + + // Preboot routes should no longer work. + await supertest(innerPrebootServer.listener).get('/preboot-get').expect(503, { + statusCode: 503, + error: 'Service Unavailable', + message: 'Kibana is shutting down and not accepting new incoming requests', + }); + await supertest(innerPrebootServer.listener).post('/preboot-post').expect(503, { + statusCode: 503, + error: 'Service Unavailable', + message: 'Kibana is shutting down and not accepting new incoming requests', + }); + + // Preboot routes should simply become unknown routes for the standard server. + await supertest(innerStandardServer.listener) + .get('/preboot-get') + .expect(404, { statusCode: 404, error: 'Not Found', message: 'Not Found' }); + await supertest(innerStandardServer.listener) + .post('/preboot-post') + .expect(404, { statusCode: 404, error: 'Not Found', message: 'Not Found' }); + + // All non-preboot routes should finally function as expected (e.g. if client tries to access any standard API). + await supertest(innerStandardServer.listener).get('/standard-get').expect(200, 'hello-get'); + await supertest(innerStandardServer.listener).post('/standard-post').expect(200, 'hello-post'); + }); +}); diff --git a/src/core/server/http/integration_tests/request.test.ts b/src/core/server/http/integration_tests/request.test.ts index ecacbf0bfa0c23..0a30bfac85f5db 100644 --- a/src/core/server/http/integration_tests/request.test.ts +++ b/src/core/server/http/integration_tests/request.test.ts @@ -30,10 +30,11 @@ const setupDeps = { executionContext: executionContextServiceMock.createInternalSetupContract(), }; -beforeEach(() => { +beforeEach(async () => { logger = loggingSystemMock.create(); server = createHttpServer({ logger }); + await server.preboot({ context: contextServiceMock.createPrebootContract() }); }); afterEach(async () => { diff --git a/src/core/server/http/integration_tests/router.test.ts b/src/core/server/http/integration_tests/router.test.ts index 1b2b0b966d3a27..5bea371d479ae5 100644 --- a/src/core/server/http/integration_tests/router.test.ts +++ b/src/core/server/http/integration_tests/router.test.ts @@ -28,9 +28,10 @@ const setupDeps = { executionContext: executionContextServiceMock.createInternalSetupContract(), }; -beforeEach(() => { +beforeEach(async () => { logger = loggingSystemMock.create(); server = createHttpServer({ logger }); + await server.preboot({ context: contextServiceMock.createPrebootContract() }); }); afterEach(async () => { diff --git a/src/core/server/http/types.ts b/src/core/server/http/types.ts index bbd296d6b1831a..7353f48b47194e 100644 --- a/src/core/server/http/types.ts +++ b/src/core/server/http/types.ts @@ -56,6 +56,109 @@ export interface HttpAuth { isAuthenticated: IsAuthenticated; } +/** + * Kibana HTTP Service provides an abstraction to work with the HTTP stack at the `preboot` stage. This functionality + * allows Kibana to serve user requests even before Kibana becomes fully operational. Only Core and `preboot` plugins + * can define HTTP routes at this stage. + * + * @example + * To handle an incoming request in your preboot plugin you should: + * - Use `@kbn/config-schema` package to create a schema to validate the request `params`, `query`, and `body`. Every incoming request will be validated against the created schema. If validation failed, the request is rejected with `400` status and `Bad request` error without calling the route's handler. + * To opt out of validating the request, specify `false`. + * ```ts + * import { schema, TypeOf } from '@kbn/config-schema'; + * const validate = { + * params: schema.object({ + * id: schema.string(), + * }), + * }; + * ``` + * + * - Declare a function to respond to incoming request. + * The function will receive `request` object containing request details: url, headers, matched route, as well as validated `params`, `query`, `body`. + * And `response` object instructing HTTP server to create HTTP response with information sent back to the client as the response body, headers, and HTTP status. + * Any exception raised during the handler call will generate `500 Server error` response and log error details for further investigation. See below for returning custom error responses. + * ```ts + * const handler = async (context: RequestHandlerContext, request: KibanaRequest, response: ResponseFactory) => { + * const data = await findObject(request.params.id); + * // creates a command to respond with 'not found' error + * if (!data) { + * return response.notFound(); + * } + * // creates a command to send found data to the client and set response headers + * return response.ok({ + * body: data, + * headers: { 'content-type': 'application/json' } + * }); + * } + * ``` + * * - Acquire `preboot` {@link IRouter} instance and register route handler for GET request to 'path/{id}' path. + * ```ts + * import { schema, TypeOf } from '@kbn/config-schema'; + * + * const validate = { + * params: schema.object({ + * id: schema.string(), + * }), + * }; + * + * httpPreboot.registerRoutes('my-plugin', (router) => { + * router.get({ path: 'path/{id}', validate }, async (context, request, response) => { + * const data = await findObject(request.params.id); + * if (!data) { + * return response.notFound(); + * } + * return response.ok({ + * body: data, + * headers: { 'content-type': 'application/json' } + * }); + * }); + * }); + * ``` + * @public + */ +export interface HttpServicePreboot { + /** + * Provides ability to acquire `preboot` {@link IRouter} instance for a particular top-level path and register handler + * functions for any number of nested routes. + * + * @remarks + * Each route can have only one handler function, which is executed when the route is matched. + * See the {@link IRouter} documentation for more information. + * + * @example + * ```ts + * registerRoutes('my-plugin', (router) => { + * // handler is called when '/my-plugin/path' resource is requested with `GET` method + * router.get({ path: '/path', validate: false }, (context, req, res) => res.ok({ content: 'ok' })); + * }); + * ``` + * @public + */ + registerRoutes(path: string, callback: (router: IRouter) => void): void; + + /** + * Access or manipulate the Kibana base path + * See {@link IBasePath}. + */ + basePath: IBasePath; +} + +/** @internal */ +export interface InternalHttpServicePreboot + extends Pick< + InternalHttpServiceSetup, + | 'auth' + | 'csp' + | 'basePath' + | 'externalUrl' + | 'registerStaticDir' + | 'registerRouteHandlerContext' + | 'server' + > { + registerRoutes(path: string, callback: (router: IRouter) => void): void; +} + /** * Kibana HTTP Service provides own abstraction for work with HTTP stack. * Plugins don't have direct access to `hapi` server and its primitives anymore. Moreover, @@ -277,11 +380,6 @@ export interface HttpServiceSetup { getServerInfo: () => HttpServerInfo; } -/** @internal */ -export interface InternalNotReadyHttpServiceSetup { - registerRoutes(path: string, callback: (router: IRouter) => void): void; -} - /** @internal */ export interface InternalHttpServiceSetup extends Omit { @@ -303,7 +401,7 @@ export interface InternalHttpServiceSetup contextName: ContextName, provider: RequestHandlerContextProvider ) => RequestHandlerContextContainer; - notReadyServer?: InternalNotReadyHttpServiceSetup; + registerPrebootRoutes(path: string, callback: (router: IRouter) => void): void; } /** @public */ diff --git a/src/core/server/http_resources/http_resources_service.mock.ts b/src/core/server/http_resources/http_resources_service.mock.ts index 3a94de15d14b95..a2ca0aa2465828 100644 --- a/src/core/server/http_resources/http_resources_service.mock.ts +++ b/src/core/server/http_resources/http_resources_service.mock.ts @@ -13,12 +13,16 @@ const createHttpResourcesMock = (): jest.Mocked => ({ register: jest.fn(), }); -function createInternalHttpResourcesSetup() { +function createInternalHttpResourcesPreboot() { return { createRegistrar: jest.fn(() => createHttpResourcesMock()), }; } +function createInternalHttpResourcesSetup() { + return createInternalHttpResourcesPreboot(); +} + function createHttpResourcesResponseFactory() { const mocked: jest.Mocked = { renderCoreApp: jest.fn(), @@ -35,6 +39,7 @@ function createHttpResourcesResponseFactory() { export const httpResourcesMock = { createRegistrar: createHttpResourcesMock, + createPrebootContract: createInternalHttpResourcesPreboot, createSetupContract: createInternalHttpResourcesSetup, createResponseFactory: createHttpResourcesResponseFactory, }; diff --git a/src/core/server/http_resources/http_resources_service.test.ts b/src/core/server/http_resources/http_resources_service.test.ts index 8b24e05fc5bf4c..33ee6cc4e37484 100644 --- a/src/core/server/http_resources/http_resources_service.test.ts +++ b/src/core/server/http_resources/http_resources_service.test.ts @@ -15,13 +15,15 @@ import { mockCoreContext } from '../core_context.mock'; import { httpServiceMock } from '../http/http_service.mock'; import { httpServerMock } from '../http/http_server.mocks'; import { renderingMock } from '../rendering/rendering_service.mock'; -import { HttpResourcesService, SetupDeps } from './http_resources_service'; +import { HttpResourcesService, PrebootDeps, SetupDeps } from './http_resources_service'; import { httpResourcesMock } from './http_resources_service.mock'; +import { HttpResources } from 'kibana/server'; const coreContext = mockCoreContext.create(); describe('HttpResources service', () => { let service: HttpResourcesService; + let prebootDeps: PrebootDeps; let setupDeps: SetupDeps; let router: jest.Mocked; const kibanaRequest = httpServerMock.createKibanaRequest(); @@ -34,6 +36,10 @@ describe('HttpResources service', () => { describe('#createRegistrar', () => { beforeEach(() => { + prebootDeps = { + http: httpServiceMock.createInternalPrebootContract(), + rendering: renderingMock.createPrebootContract(), + }; setupDeps = { http: httpServiceMock.createInternalSetupContract(), rendering: renderingMock.createSetupContract(), @@ -42,221 +48,228 @@ describe('HttpResources service', () => { router = httpServiceMock.createRouter(); }); - describe('register', () => { - describe('renderCoreApp', () => { - it('formats successful response', async () => { - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderCoreApp(); - }); - const [[, routeHandler]] = router.get.mock.calls; - - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); - expect(setupDeps.rendering.render).toHaveBeenCalledWith( - kibanaRequest, - context.core.uiSettings.client, - { - includeUserSettings: true, - vars: { - apmConfig, - }, - } - ); + function runRegisterTestSuite( + name: string, + initializer: () => Promise, + getDeps: () => PrebootDeps | SetupDeps + ) { + describe(`${name} register`, () => { + const routeConfig: RouteConfig = { path: '/', validate: false }; + let register: HttpResources['register']; + beforeEach(async () => { + register = await initializer(); }); - it('can attach headers, except the CSP header', async () => { - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderCoreApp({ - headers: { - 'content-security-policy': "script-src 'unsafe-eval'", - 'x-kibana': '42', - }, + describe('renderCoreApp', () => { + it('formats successful response', async () => { + register(routeConfig, async (ctx, req, res) => { + return res.renderCoreApp(); }); - }); + const [[, routeHandler]] = router.get.mock.calls; - const [[, routeHandler]] = router.get.mock.calls; + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); + expect(getDeps().rendering.render).toHaveBeenCalledWith( + kibanaRequest, + context.core.uiSettings.client, + { + includeUserSettings: true, + vars: { + apmConfig, + }, + } + ); + }); - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); + it('can attach headers, except the CSP header', async () => { + register(routeConfig, async (ctx, req, res) => { + return res.renderCoreApp({ + headers: { + 'content-security-policy': "script-src 'unsafe-eval'", + 'x-kibana': '42', + }, + }); + }); - expect(responseFactory.ok).toHaveBeenCalledWith({ - body: '', - headers: { - 'x-kibana': '42', - 'content-security-policy': - "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", - }, - }); - }); - }); - describe('renderAnonymousCoreApp', () => { - it('formats successful response', async () => { - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderAnonymousCoreApp(); - }); - const [[, routeHandler]] = router.get.mock.calls; + const [[, routeHandler]] = router.get.mock.calls; - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); - expect(setupDeps.rendering.render).toHaveBeenCalledWith( - kibanaRequest, - context.core.uiSettings.client, - { - includeUserSettings: false, - vars: { - apmConfig, - }, - } - ); - }); + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); - it('can attach headers, except the CSP header', async () => { - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderAnonymousCoreApp({ + expect(responseFactory.ok).toHaveBeenCalledWith({ + body: '', headers: { - 'content-security-policy': "script-src 'unsafe-eval'", 'x-kibana': '42', + 'content-security-policy': + "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", }, }); }); + }); + describe('renderAnonymousCoreApp', () => { + it('formats successful response', async () => { + register(routeConfig, async (ctx, req, res) => { + return res.renderAnonymousCoreApp(); + }); + const [[, routeHandler]] = router.get.mock.calls; - const [[, routeHandler]] = router.get.mock.calls; + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); + expect(getDeps().rendering.render).toHaveBeenCalledWith( + kibanaRequest, + context.core.uiSettings.client, + { + includeUserSettings: false, + vars: { + apmConfig, + }, + } + ); + }); - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); + it('can attach headers, except the CSP header', async () => { + register(routeConfig, async (ctx, req, res) => { + return res.renderAnonymousCoreApp({ + headers: { + 'content-security-policy': "script-src 'unsafe-eval'", + 'x-kibana': '42', + }, + }); + }); - expect(responseFactory.ok).toHaveBeenCalledWith({ - body: '', - headers: { - 'x-kibana': '42', - 'content-security-policy': - "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", - }, - }); - }); - }); - describe('renderHtml', () => { - it('formats successful response', async () => { - const htmlBody = ''; - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderHtml({ body: htmlBody }); - }); - const [[, routeHandler]] = router.get.mock.calls; + const [[, routeHandler]] = router.get.mock.calls; + + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); - expect(responseFactory.ok).toHaveBeenCalledWith({ - body: htmlBody, - headers: { - 'content-type': 'text/html', - 'content-security-policy': - "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", - }, + expect(responseFactory.ok).toHaveBeenCalledWith({ + body: '', + headers: { + 'x-kibana': '42', + 'content-security-policy': + "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", + }, + }); }); }); + describe('renderHtml', () => { + it('formats successful response', async () => { + const htmlBody = ''; + register(routeConfig, async (ctx, req, res) => { + return res.renderHtml({ body: htmlBody }); + }); + const [[, routeHandler]] = router.get.mock.calls; - it('can attach headers, except the CSP & "content-type" headers', async () => { - const htmlBody = ''; - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderHtml({ + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); + expect(responseFactory.ok).toHaveBeenCalledWith({ body: htmlBody, headers: { - 'content-type': 'text/html5', - 'content-security-policy': "script-src 'unsafe-eval'", - 'x-kibana': '42', + 'content-type': 'text/html', + 'content-security-policy': + "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", }, }); }); - const [[, routeHandler]] = router.get.mock.calls; + it('can attach headers, except the CSP & "content-type" headers', async () => { + const htmlBody = ''; + register(routeConfig, async (ctx, req, res) => { + return res.renderHtml({ + body: htmlBody, + headers: { + 'content-type': 'text/html5', + 'content-security-policy': "script-src 'unsafe-eval'", + 'x-kibana': '42', + }, + }); + }); - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); + const [[, routeHandler]] = router.get.mock.calls; - expect(responseFactory.ok).toHaveBeenCalledWith({ - body: htmlBody, - headers: { - 'content-type': 'text/html', - 'x-kibana': '42', - 'content-security-policy': - "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", - }, - }); - }); - }); - describe('renderJs', () => { - it('formats successful response', async () => { - const jsBody = 'alert(1);'; - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderJs({ body: jsBody }); - }); - const [[, routeHandler]] = router.get.mock.calls; + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); - expect(responseFactory.ok).toHaveBeenCalledWith({ - body: jsBody, - headers: { - 'content-type': 'text/javascript', - 'content-security-policy': - "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", - }, + expect(responseFactory.ok).toHaveBeenCalledWith({ + body: htmlBody, + headers: { + 'content-type': 'text/html', + 'x-kibana': '42', + 'content-security-policy': + "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", + }, + }); }); }); + describe('renderJs', () => { + it('formats successful response', async () => { + const jsBody = 'alert(1);'; + register(routeConfig, async (ctx, req, res) => { + return res.renderJs({ body: jsBody }); + }); + const [[, routeHandler]] = router.get.mock.calls; - it('can attach headers, except the CSP & "content-type" headers', async () => { - const jsBody = 'alert(1);'; - const routeConfig: RouteConfig = { path: '/', validate: false }; - const { createRegistrar } = await service.setup(setupDeps); - const { register } = createRegistrar(router); - register(routeConfig, async (ctx, req, res) => { - return res.renderJs({ + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); + expect(responseFactory.ok).toHaveBeenCalledWith({ body: jsBody, headers: { - 'content-type': 'text/html', - 'content-security-policy': "script-src 'unsafe-eval'", - 'x-kibana': '42', + 'content-type': 'text/javascript', + 'content-security-policy': + "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", }, }); }); - const [[, routeHandler]] = router.get.mock.calls; + it('can attach headers, except the CSP & "content-type" headers', async () => { + const jsBody = 'alert(1);'; + register(routeConfig, async (ctx, req, res) => { + return res.renderJs({ + body: jsBody, + headers: { + 'content-type': 'text/html', + 'content-security-policy': "script-src 'unsafe-eval'", + 'x-kibana': '42', + }, + }); + }); + + const [[, routeHandler]] = router.get.mock.calls; - const responseFactory = httpResourcesMock.createResponseFactory(); - await routeHandler(context, kibanaRequest, responseFactory); + const responseFactory = httpResourcesMock.createResponseFactory(); + await routeHandler(context, kibanaRequest, responseFactory); - expect(responseFactory.ok).toHaveBeenCalledWith({ - body: jsBody, - headers: { - 'content-type': 'text/javascript', - 'x-kibana': '42', - 'content-security-policy': - "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", - }, + expect(responseFactory.ok).toHaveBeenCalledWith({ + body: jsBody, + headers: { + 'content-type': 'text/javascript', + 'x-kibana': '42', + 'content-security-policy': + "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", + }, + }); }); }); }); - }); + } + + runRegisterTestSuite( + '#preboot', + async () => { + const { createRegistrar } = await service.preboot(prebootDeps); + return createRegistrar(router).register; + }, + () => prebootDeps + ); + + runRegisterTestSuite( + '#setup', + async () => { + await service.preboot(prebootDeps); + const { createRegistrar } = await service.setup(setupDeps); + return createRegistrar(router).register; + }, + () => setupDeps + ); }); }); diff --git a/src/core/server/http_resources/http_resources_service.ts b/src/core/server/http_resources/http_resources_service.ts index 44caa456e99559..6c295b152af3c2 100644 --- a/src/core/server/http_resources/http_resources_service.ts +++ b/src/core/server/http_resources/http_resources_service.ts @@ -15,10 +15,11 @@ import { InternalHttpServiceSetup, KibanaRequest, KibanaResponseFactory, + InternalHttpServicePreboot, } from '../http'; import { Logger } from '../logging'; -import { InternalRenderingServiceSetup } from '../rendering'; +import { InternalRenderingServicePreboot, InternalRenderingServiceSetup } from '../rendering'; import { CoreService } from '../../types'; import { @@ -31,6 +32,11 @@ import { } from './types'; import { getApmConfig } from './get_apm_config'; +export interface PrebootDeps { + http: InternalHttpServicePreboot; + rendering: InternalRenderingServicePreboot; +} + export interface SetupDeps { http: InternalHttpServiceSetup; rendering: InternalRenderingServiceSetup; @@ -43,6 +49,13 @@ export class HttpResourcesService implements CoreService( route: RouteConfig, @@ -71,7 +84,7 @@ export class HttpResourcesService implements CoreService { }, plugins: { initialize: false }, }); + await root.preboot(); }, 30000); afterEach(async () => { diff --git a/src/core/server/http_resources/types.ts b/src/core/server/http_resources/types.ts index 3333574038ec7c..1ec02272d151f7 100644 --- a/src/core/server/http_resources/types.ts +++ b/src/core/server/http_resources/types.ts @@ -84,10 +84,16 @@ export type HttpResourcesRequestHandler< * Allows to configure HTTP response parameters * @internal */ -export interface InternalHttpResourcesSetup { +export interface InternalHttpResourcesPreboot { createRegistrar(router: IRouter): HttpResources; } +/** + * Allows to configure HTTP response parameters + * @internal + */ +export type InternalHttpResourcesSetup = InternalHttpResourcesPreboot; + /** * HttpResources service is responsible for serving static & dynamic assets for Kibana application via HTTP. * Provides API allowing plug-ins to respond with: diff --git a/src/core/server/i18n/i18n_service.mock.ts b/src/core/server/i18n/i18n_service.mock.ts index 29859e95b63b2a..a199acd00eff5f 100644 --- a/src/core/server/i18n/i18n_service.mock.ts +++ b/src/core/server/i18n/i18n_service.mock.ts @@ -25,6 +25,7 @@ type I18nServiceContract = PublicMethodsOf; const createMock = () => { const mock: jest.Mocked = { + preboot: jest.fn(), setup: jest.fn(), }; diff --git a/src/core/server/i18n/i18n_service.test.ts b/src/core/server/i18n/i18n_service.test.ts index 913d5ee4aaa99f..ad87b371aca338 100644 --- a/src/core/server/i18n/i18n_service.test.ts +++ b/src/core/server/i18n/i18n_service.test.ts @@ -17,7 +17,7 @@ import { I18nService } from './i18n_service'; import { configServiceMock } from '../config/mocks'; import { mockCoreContext } from '../core_context.mock'; -import { httpServiceMock } from '../http/http_service.mock'; +import { httpServiceMock } from '../mocks'; const getConfigService = (locale = 'en') => { const configService = configServiceMock.create(); @@ -35,7 +35,8 @@ const getConfigService = (locale = 'en') => { describe('I18nService', () => { let service: I18nService; let configService: ReturnType; - let http: ReturnType; + let httpPreboot: ReturnType; + let httpSetup: ReturnType; beforeEach(() => { jest.clearAllMocks(); @@ -44,15 +45,60 @@ describe('I18nService', () => { const coreContext = mockCoreContext.create({ configService }); service = new I18nService(coreContext); - http = httpServiceMock.createInternalSetupContract(); + httpPreboot = httpServiceMock.createInternalPrebootContract(); + httpPreboot.registerRoutes.mockImplementation((type, callback) => + callback(httpServiceMock.createRouter()) + ); + httpSetup = httpServiceMock.createInternalSetupContract(); + }); + + describe('#preboot', () => { + it('calls `getKibanaTranslationFiles` with the correct parameters', async () => { + getKibanaTranslationFilesMock.mockResolvedValue([]); + + const pluginPaths = ['/pathA', '/pathB']; + await service.preboot({ pluginPaths, http: httpPreboot }); + + expect(getKibanaTranslationFilesMock).toHaveBeenCalledTimes(1); + expect(getKibanaTranslationFilesMock).toHaveBeenCalledWith('en', pluginPaths); + }); + + it('calls `initTranslations` with the correct parameters', async () => { + const translationFiles = ['/path/to/file', 'path/to/another/file']; + getKibanaTranslationFilesMock.mockResolvedValue(translationFiles); + + await service.preboot({ pluginPaths: [], http: httpPreboot }); + + expect(initTranslationsMock).toHaveBeenCalledTimes(1); + expect(initTranslationsMock).toHaveBeenCalledWith('en', translationFiles); + }); + + it('calls `registerRoutesMock` with the correct parameters', async () => { + await service.preboot({ pluginPaths: [], http: httpPreboot }); + + expect(registerRoutesMock).toHaveBeenCalledTimes(1); + expect(registerRoutesMock).toHaveBeenCalledWith({ + locale: 'en', + router: expect.any(Object), + }); + }); }); describe('#setup', () => { + beforeEach(async () => { + await service.preboot({ pluginPaths: ['/pathPrebootA'], http: httpPreboot }); + + // Reset mocks that were used in the `preboot`. + getKibanaTranslationFilesMock.mockClear(); + initTranslationsMock.mockClear(); + registerRoutesMock.mockClear(); + }); + it('calls `getKibanaTranslationFiles` with the correct parameters', async () => { getKibanaTranslationFilesMock.mockResolvedValue([]); const pluginPaths = ['/pathA', '/pathB']; - await service.setup({ pluginPaths, http }); + await service.setup({ pluginPaths, http: httpSetup }); expect(getKibanaTranslationFilesMock).toHaveBeenCalledTimes(1); expect(getKibanaTranslationFilesMock).toHaveBeenCalledWith('en', pluginPaths); @@ -62,14 +108,14 @@ describe('I18nService', () => { const translationFiles = ['/path/to/file', 'path/to/another/file']; getKibanaTranslationFilesMock.mockResolvedValue(translationFiles); - await service.setup({ pluginPaths: [], http }); + await service.setup({ pluginPaths: [], http: httpSetup }); expect(initTranslationsMock).toHaveBeenCalledTimes(1); expect(initTranslationsMock).toHaveBeenCalledWith('en', translationFiles); }); it('calls `registerRoutesMock` with the correct parameters', async () => { - await service.setup({ pluginPaths: [], http }); + await service.setup({ pluginPaths: [], http: httpSetup }); expect(registerRoutesMock).toHaveBeenCalledTimes(1); expect(registerRoutesMock).toHaveBeenCalledWith({ @@ -82,7 +128,10 @@ describe('I18nService', () => { const translationFiles = ['/path/to/file', 'path/to/another/file']; getKibanaTranslationFilesMock.mockResolvedValue(translationFiles); - const { getLocale, getTranslationFiles } = await service.setup({ pluginPaths: [], http }); + const { getLocale, getTranslationFiles } = await service.setup({ + pluginPaths: [], + http: httpSetup, + }); expect(getLocale()).toEqual('en'); expect(getTranslationFiles()).toEqual(translationFiles); diff --git a/src/core/server/i18n/i18n_service.ts b/src/core/server/i18n/i18n_service.ts index 0dffd8934a8cab..02a809b3c2c36b 100644 --- a/src/core/server/i18n/i18n_service.ts +++ b/src/core/server/i18n/i18n_service.ts @@ -10,12 +10,17 @@ import { take } from 'rxjs/operators'; import { Logger } from '../logging'; import { IConfigService } from '../config'; import { CoreContext } from '../core_context'; -import { InternalHttpServiceSetup } from '../http'; +import { InternalHttpServicePreboot, InternalHttpServiceSetup } from '../http'; import { config as i18nConfigDef, I18nConfigType } from './i18n_config'; import { getKibanaTranslationFiles } from './get_kibana_translation_files'; import { initTranslations } from './init_translations'; import { registerRoutes } from './routes'; +interface PrebootDeps { + http: InternalHttpServicePreboot; + pluginPaths: string[]; +} + interface SetupDeps { http: InternalHttpServiceSetup; pluginPaths: string[]; @@ -45,7 +50,24 @@ export class I18nService { this.configService = coreContext.configService; } + public async preboot({ pluginPaths, http }: PrebootDeps) { + const { locale } = await this.initTranslations(pluginPaths); + http.registerRoutes('', (router) => registerRoutes({ router, locale })); + } + public async setup({ pluginPaths, http }: SetupDeps): Promise { + const { locale, translationFiles } = await this.initTranslations(pluginPaths); + + const router = http.createRouter(''); + registerRoutes({ router, locale }); + + return { + getLocale: () => locale, + getTranslationFiles: () => translationFiles, + }; + } + + private async initTranslations(pluginPaths: string[]) { const i18nConfig = await this.configService .atPath(i18nConfigDef.path) .pipe(take(1)) @@ -59,12 +81,6 @@ export class I18nService { this.log.debug(`Using translation files: [${translationFiles.join(', ')}]`); await initTranslations(locale, translationFiles); - const router = http.createRouter(''); - registerRoutes({ router, locale }); - - return { - getLocale: () => locale, - getTranslationFiles: () => translationFiles, - }; + return { locale, translationFiles }; } } diff --git a/src/core/server/index.ts b/src/core/server/index.ts index e502fff6c69bf9..c77a3a967364c5 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -35,8 +35,9 @@ import { configSchema as elasticsearchConfigSchema, ElasticsearchServiceStart, IScopedClusterClient, + ElasticsearchServicePreboot, } from './elasticsearch'; -import { HttpServiceSetup, HttpServiceStart } from './http'; +import { HttpServicePreboot, HttpServiceSetup, HttpServiceStart } from './http'; import { HttpResources } from './http_resources'; import { PluginsServiceSetup, PluginsServiceStart, PluginOpaqueId } from './plugins'; @@ -58,7 +59,7 @@ import { AppenderConfigType, appendersSchema, LoggingServiceSetup } from './logg import { CoreUsageDataStart } from './core_usage_data'; import { I18nServiceSetup } from './i18n'; import { DeprecationsServiceSetup } from './deprecations'; -// Because of #79265 we need to explicity import, then export these types for +// Because of #79265 we need to explicitly import, then export these types for // scripts/telemetry_check.js to work as expected import { CoreUsageStats, @@ -68,6 +69,9 @@ import { CoreEnvironmentUsageData, CoreServicesUsageData, } from './core_usage_data'; +import { PrebootServicePreboot } from './preboot'; + +export type { PrebootServicePreboot } from './preboot'; export type { CoreUsageStats, @@ -125,6 +129,7 @@ export type { LegacyElasticsearchClientConfig, LegacyElasticsearchError, LegacyElasticsearchErrorHelpers, + ElasticsearchServicePreboot, ElasticsearchServiceSetup, ElasticsearchServiceStart, ElasticsearchStatusMeta, @@ -143,6 +148,7 @@ export type { ShardsResponse, GetResponse, DeleteDocumentResponse, + ElasticsearchConfigPreboot, } from './elasticsearch'; export type { @@ -179,6 +185,7 @@ export type { HttpResponseOptions, HttpResponsePayload, HttpServerInfo, + HttpServicePreboot, HttpServiceSetup, HttpServiceStart, ErrorHttpResponseOptions, @@ -260,8 +267,11 @@ export type { AppenderConfigType, } from './logging'; +export { PluginType } from './plugins'; + export type { DiscoveredPlugin, + PrebootPlugin, Plugin, AsyncPlugin, PluginConfigDescriptor, @@ -468,7 +478,20 @@ export interface RequestHandlerContext { } /** - * Context passed to the plugins `setup` method. + * Context passed to the `setup` method of `preboot` plugins. + * @public + */ +export interface CorePreboot { + /** {@link ElasticsearchServicePreboot} */ + elasticsearch: ElasticsearchServicePreboot; + /** {@link HttpServicePreboot} */ + http: HttpServicePreboot; + /** {@link PrebootServicePreboot} */ + preboot: PrebootServicePreboot; +} + +/** + * Context passed to the `setup` method of `standard` plugins. * * @typeParam TPluginsStart - the type of the consuming plugin's start dependencies. Should be the same * as the consuming {@link Plugin}'s `TPluginsStart` type. Used by `getStartServices`. diff --git a/src/core/server/internal_types.ts b/src/core/server/internal_types.ts index f3253e32aa08e8..540670274f4167 100644 --- a/src/core/server/internal_types.ts +++ b/src/core/server/internal_types.ts @@ -10,23 +10,32 @@ import { Type } from '@kbn/config-schema'; import { CapabilitiesSetup, CapabilitiesStart } from './capabilities'; import { ConfigDeprecationProvider } from './config'; -import { ContextSetup } from './context'; +import { InternalContextPreboot, ContextSetup } from './context'; import { + InternalElasticsearchServicePreboot, InternalElasticsearchServiceSetup, InternalElasticsearchServiceStart, } from './elasticsearch'; -import { InternalHttpServiceSetup, InternalHttpServiceStart } from './http'; +import { + InternalHttpServicePreboot, + InternalHttpServiceSetup, + InternalHttpServiceStart, +} from './http'; import { InternalSavedObjectsServiceSetup, InternalSavedObjectsServiceStart, } from './saved_objects'; -import { InternalUiSettingsServiceSetup, InternalUiSettingsServiceStart } from './ui_settings'; +import { + InternalUiSettingsServicePreboot, + InternalUiSettingsServiceSetup, + InternalUiSettingsServiceStart, +} from './ui_settings'; import { InternalEnvironmentServiceSetup } from './environment'; import { InternalMetricsServiceSetup, InternalMetricsServiceStart } from './metrics'; import { InternalRenderingServiceSetup } from './rendering'; -import { InternalHttpResourcesSetup } from './http_resources'; +import { InternalHttpResourcesPreboot, InternalHttpResourcesSetup } from './http_resources'; import { InternalStatusServiceSetup } from './status'; -import { InternalLoggingServiceSetup } from './logging'; +import { InternalLoggingServicePreboot, InternalLoggingServiceSetup } from './logging'; import { CoreUsageDataStart } from './core_usage_data'; import { I18nServiceSetup } from './i18n'; import { InternalDeprecationsServiceSetup } from './deprecations'; @@ -34,6 +43,18 @@ import type { InternalExecutionContextSetup, InternalExecutionContextStart, } from './execution_context'; +import { InternalPrebootServicePreboot } from './preboot'; + +/** @internal */ +export interface InternalCorePreboot { + context: InternalContextPreboot; + http: InternalHttpServicePreboot; + elasticsearch: InternalElasticsearchServicePreboot; + uiSettings: InternalUiSettingsServicePreboot; + httpResources: InternalHttpResourcesPreboot; + logging: InternalLoggingServicePreboot; + preboot: InternalPrebootServicePreboot; +} /** @internal */ export interface InternalCoreSetup { diff --git a/src/core/server/legacy/integration_tests/logging.test.ts b/src/core/server/legacy/integration_tests/logging.test.ts index 88c45962ce4a68..d8f9f035f44be3 100644 --- a/src/core/server/legacy/integration_tests/logging.test.ts +++ b/src/core/server/legacy/integration_tests/logging.test.ts @@ -67,6 +67,7 @@ describe('logging service', () => { beforeAll(async () => { root = createRoot(); + await root.preboot(); await root.setup(); await root.start(); }, 30000); @@ -119,6 +120,7 @@ describe('logging service', () => { it('"silent": true', async () => { root = createRoot({ silent: true }); + await root.preboot(); await root.setup(); await root.start(); @@ -150,6 +152,7 @@ describe('logging service', () => { it('"quiet": true', async () => { root = createRoot({ quiet: true }); + await root.preboot(); await root.setup(); await root.start(); @@ -187,6 +190,7 @@ describe('logging service', () => { it('"verbose": true', async () => { root = createRoot({ verbose: true }); + await root.preboot(); await root.setup(); await root.start(); diff --git a/src/core/server/logging/index.ts b/src/core/server/logging/index.ts index 9d17b289bfa4c2..ba6aec5f128c1f 100644 --- a/src/core/server/logging/index.ts +++ b/src/core/server/logging/index.ts @@ -32,6 +32,10 @@ export type { export { LoggingSystem } from './logging_system'; export type { ILoggingSystem } from './logging_system'; export { LoggingService } from './logging_service'; -export type { InternalLoggingServiceSetup, LoggingServiceSetup } from './logging_service'; +export type { + InternalLoggingServicePreboot, + InternalLoggingServiceSetup, + LoggingServiceSetup, +} from './logging_service'; export { appendersSchema } from './appenders/appenders'; export type { AppenderConfigType } from './appenders/appenders'; diff --git a/src/core/server/logging/integration_tests/logging.test.ts b/src/core/server/logging/integration_tests/logging.test.ts index b4eb98546de21b..ade10fc1c0257c 100644 --- a/src/core/server/logging/integration_tests/logging.test.ts +++ b/src/core/server/logging/integration_tests/logging.test.ts @@ -49,6 +49,7 @@ describe('logging service', () => { mockConsoleLog = jest.spyOn(global.console, 'log'); root = createRoot(); + await root.preboot(); await root.setup(); }, 30000); @@ -151,6 +152,7 @@ describe('logging service', () => { mockConsoleLog = jest.spyOn(global.console, 'log'); root = kbnTestServer.createRoot(); + await root.preboot(); setup = await root.setup(); setup.logging.configure(['plugins', 'myplugin'], loggingConfig$); }, 30000); diff --git a/src/core/server/logging/integration_tests/rolling_file_appender.test.ts b/src/core/server/logging/integration_tests/rolling_file_appender.test.ts index b40ce7a4e7b0e3..b560748026ace1 100644 --- a/src/core/server/logging/integration_tests/rolling_file_appender.test.ts +++ b/src/core/server/logging/integration_tests/rolling_file_appender.test.ts @@ -79,6 +79,7 @@ describe('RollingFileAppender', () => { pattern: '.%i', }, }); + await root.preboot(); await root.setup(); const logger = root.logger.get('test.rolling.file'); @@ -124,6 +125,7 @@ describe('RollingFileAppender', () => { pattern: '-%i', }, }); + await root.preboot(); await root.setup(); const logger = root.logger.get('test.rolling.file'); @@ -174,6 +176,7 @@ describe('RollingFileAppender', () => { pattern: '-%i', }, }); + await root.preboot(); await root.setup(); const logger = root.logger.get('test.rolling.file'); diff --git a/src/core/server/logging/logging_service.mock.ts b/src/core/server/logging/logging_service.mock.ts index 5f91c7b8734b89..75d358c8cbe6e1 100644 --- a/src/core/server/logging/logging_service.mock.ts +++ b/src/core/server/logging/logging_service.mock.ts @@ -12,8 +12,13 @@ import { LoggingService, LoggingServiceSetup, InternalLoggingServiceSetup, + InternalLoggingServicePreboot, } from './logging_service'; +const createInternalPrebootMock = (): jest.Mocked => ({ + configure: jest.fn(), +}); + const createInternalSetupMock = (): jest.Mocked => ({ configure: jest.fn(), }); @@ -25,11 +30,13 @@ const createSetupMock = (): jest.Mocked => ({ type LoggingServiceContract = PublicMethodsOf; const createMock = (): jest.Mocked => { const service: jest.Mocked = { + preboot: jest.fn(), setup: jest.fn(), start: jest.fn(), stop: jest.fn(), }; + service.preboot.mockReturnValue(createInternalPrebootMock()); service.setup.mockReturnValue(createInternalSetupMock()); return service; @@ -38,5 +45,6 @@ const createMock = (): jest.Mocked => { export const loggingServiceMock = { create: createMock, createSetupContract: createSetupMock, + createInternalPrebootContract: createInternalPrebootMock, createInternalSetupContract: createInternalSetupMock, }; diff --git a/src/core/server/logging/logging_service.test.ts b/src/core/server/logging/logging_service.test.ts index 341a04736b87a4..9817f08c59bf85 100644 --- a/src/core/server/logging/logging_service.test.ts +++ b/src/core/server/logging/logging_service.test.ts @@ -8,83 +8,102 @@ import { of, Subject } from 'rxjs'; -import { LoggingService, InternalLoggingServiceSetup } from './logging_service'; +import { + LoggingService, + InternalLoggingServiceSetup, + InternalLoggingServicePreboot, +} from './logging_service'; import { loggingSystemMock } from './logging_system.mock'; import { LoggerContextConfigType } from './logging_config'; describe('LoggingService', () => { let loggingSystem: ReturnType; let service: LoggingService; - let setup: InternalLoggingServiceSetup; + let preboot: InternalLoggingServicePreboot; beforeEach(() => { loggingSystem = loggingSystemMock.create(); service = new LoggingService({ logger: loggingSystem.asLoggerFactory() } as any); - setup = service.setup({ loggingSystem }); + preboot = service.preboot({ loggingSystem }); }); afterEach(() => { service.stop(); }); - describe('setup', () => { - it('forwards configuration changes to logging system', () => { - const config1: LoggerContextConfigType = { - appenders: new Map(), - loggers: [{ name: 'subcontext', appenders: ['console'], level: 'warn' }], - }; - const config2: LoggerContextConfigType = { - appenders: new Map(), - loggers: [{ name: 'subcontext', appenders: ['default'], level: 'all' }], - }; + function runTestSuite( + testSuiteName: string, + getContract: () => InternalLoggingServicePreboot | InternalLoggingServiceSetup + ) { + describe(testSuiteName, () => { + it('forwards configuration changes to logging system', async () => { + const config1: LoggerContextConfigType = { + appenders: new Map(), + loggers: [{ name: 'subcontext', appenders: ['console'], level: 'warn' }], + }; + const config2: LoggerContextConfigType = { + appenders: new Map(), + loggers: [{ name: 'subcontext', appenders: ['default'], level: 'all' }], + }; - setup.configure(['test', 'context'], of(config1, config2)); - expect(loggingSystem.setContextConfig).toHaveBeenNthCalledWith( - 1, - ['test', 'context'], - config1 - ); - expect(loggingSystem.setContextConfig).toHaveBeenNthCalledWith( - 2, - ['test', 'context'], - config2 - ); - }); + getContract().configure(['test', 'context'], of(config1, config2)); + expect(loggingSystem.setContextConfig).toHaveBeenNthCalledWith( + 1, + ['test', 'context'], + config1 + ); + expect(loggingSystem.setContextConfig).toHaveBeenNthCalledWith( + 2, + ['test', 'context'], + config2 + ); + }); - it('stops forwarding first observable when called a second time', () => { - const updates$ = new Subject(); - const config1: LoggerContextConfigType = { - appenders: new Map(), - loggers: [{ name: 'subcontext', appenders: ['console'], level: 'warn' }], - }; - const config2: LoggerContextConfigType = { - appenders: new Map(), - loggers: [{ name: 'subcontext', appenders: ['default'], level: 'all' }], - }; + it('stops forwarding first observable when called a second time', () => { + const updates$ = new Subject(); + const config1: LoggerContextConfigType = { + appenders: new Map(), + loggers: [{ name: 'subcontext', appenders: ['console'], level: 'warn' }], + }; + const config2: LoggerContextConfigType = { + appenders: new Map(), + loggers: [{ name: 'subcontext', appenders: ['default'], level: 'all' }], + }; - setup.configure(['test', 'context'], updates$); - setup.configure(['test', 'context'], of(config1)); - updates$.next(config2); - expect(loggingSystem.setContextConfig).toHaveBeenNthCalledWith( - 1, - ['test', 'context'], - config1 - ); - expect(loggingSystem.setContextConfig).not.toHaveBeenCalledWith(['test', 'context'], config2); + const contract = getContract(); + contract.configure(['test', 'context'], updates$); + contract.configure(['test', 'context'], of(config1)); + updates$.next(config2); + expect(loggingSystem.setContextConfig).toHaveBeenNthCalledWith( + 1, + ['test', 'context'], + config1 + ); + expect(loggingSystem.setContextConfig).not.toHaveBeenCalledWith( + ['test', 'context'], + config2 + ); + }); }); - }); - describe('stop', () => { - it('stops forwarding updates to logging system', () => { - const updates$ = new Subject(); - const config1: LoggerContextConfigType = { - appenders: new Map(), - loggers: [{ name: 'subcontext', appenders: ['console'], level: 'warn' }], - }; + describe(`stop after ${testSuiteName}`, () => { + it('stops forwarding updates to logging system', () => { + const updates$ = new Subject(); + const config1: LoggerContextConfigType = { + appenders: new Map(), + loggers: [{ name: 'subcontext', appenders: ['console'], level: 'warn' }], + }; - setup.configure(['test', 'context'], updates$); - service.stop(); - updates$.next(config1); - expect(loggingSystem.setContextConfig).not.toHaveBeenCalledWith(['test', 'context'], config1); + getContract().configure(['test', 'context'], updates$); + service.stop(); + updates$.next(config1); + expect(loggingSystem.setContextConfig).not.toHaveBeenCalledWith( + ['test', 'context'], + config1 + ); + }); }); - }); + } + + runTestSuite('preboot', () => preboot); + runTestSuite('setup', () => service.setup()); }); diff --git a/src/core/server/logging/logging_service.ts b/src/core/server/logging/logging_service.ts index f5a4717fdbfaf4..6c3eee4981725d 100644 --- a/src/core/server/logging/logging_service.ts +++ b/src/core/server/logging/logging_service.ts @@ -42,11 +42,14 @@ export interface LoggingServiceSetup { } /** @internal */ -export interface InternalLoggingServiceSetup { +export interface InternalLoggingServicePreboot { configure(contextParts: string[], config$: Observable): void; } -interface SetupDeps { +/** @internal */ +export type InternalLoggingServiceSetup = InternalLoggingServicePreboot; + +interface PrebootDeps { loggingSystem: ILoggingSystem; } @@ -54,13 +57,14 @@ interface SetupDeps { export class LoggingService implements CoreService { private readonly subscriptions = new Map(); private readonly log: Logger; + private internalPreboot?: InternalLoggingServicePreboot; constructor(coreContext: CoreContext) { this.log = coreContext.logger.get('logging'); } - public setup({ loggingSystem }: SetupDeps) { - return { + public preboot({ loggingSystem }: PrebootDeps) { + this.internalPreboot = { configure: (contextParts: string[], config$: Observable) => { const contextName = LoggingConfig.getLoggerContext(contextParts); this.log.debug(`Setting custom config for context [${contextName}]`); @@ -80,6 +84,14 @@ export class LoggingService implements CoreService ); }, }; + + return this.internalPreboot; + } + + public setup() { + return { + configure: this.internalPreboot!.configure, + }; } public start() {} diff --git a/src/core/server/metrics/integration_tests/server_collector.test.ts b/src/core/server/metrics/integration_tests/server_collector.test.ts index 19e3e6a6c68f98..93589648ca0ae4 100644 --- a/src/core/server/metrics/integration_tests/server_collector.test.ts +++ b/src/core/server/metrics/integration_tests/server_collector.test.ts @@ -29,6 +29,7 @@ describe('ServerMetricsCollector', () => { beforeEach(async () => { server = createHttpServer(); + await server.preboot({ context: contextServiceMock.createPrebootContract() }); const contextSetup = contextServiceMock.createSetupContract(); const httpSetup = await server.setup({ context: contextSetup, diff --git a/src/core/server/mocks.ts b/src/core/server/mocks.ts index ff844f44aede06..f423e40fbcaf98 100644 --- a/src/core/server/mocks.ts +++ b/src/core/server/mocks.ts @@ -10,7 +10,13 @@ import { of } from 'rxjs'; import { duration } from 'moment'; import { ByteSizeValue } from '@kbn/config-schema'; import type { MockedKeys } from '@kbn/utility-types/jest'; -import { PluginInitializerContext, CoreSetup, CoreStart, StartServicesAccessor } from '.'; +import { + PluginInitializerContext, + CoreSetup, + CoreStart, + StartServicesAccessor, + CorePreboot, +} from '.'; import { loggingSystemMock } from './logging/logging_system.mock'; import { loggingServiceMock } from './logging/logging_service.mock'; import { elasticsearchServiceMock } from './elasticsearch/elasticsearch_service.mock'; @@ -31,6 +37,7 @@ import { coreUsageDataServiceMock } from './core_usage_data/core_usage_data_serv import { i18nServiceMock } from './i18n/i18n_service.mock'; import { deprecationsServiceMock } from './deprecations/deprecations_service.mock'; import { executionContextServiceMock } from './execution_context/execution_context_service.mock'; +import { prebootServiceMock } from './preboot/preboot_service.mock'; export { configServiceMock } from './config/mocks'; export { httpServerMock } from './http/http_server.mocks'; @@ -106,6 +113,7 @@ function pluginInitializerContextMock(config: T = {} as T) { dist: false, }, instanceUuid: 'instance-uuid', + configs: ['/some/path/to/config/kibana.yml'], }, config: pluginInitializerContextConfigMock(config), }; @@ -113,6 +121,20 @@ function pluginInitializerContextMock(config: T = {} as T) { return mock; } +type CorePrebootMockType = MockedKeys & { + elasticsearch: ReturnType; +}; + +function createCorePrebootMock() { + const mock: CorePrebootMockType = { + elasticsearch: elasticsearchServiceMock.createPreboot(), + http: httpServiceMock.createPrebootContract(), + preboot: prebootServiceMock.createPrebootContract(), + }; + + return mock; +} + type CoreSetupMockType = MockedKeys & { elasticsearch: ReturnType; getStartServices: jest.MockedFunction>; @@ -170,6 +192,19 @@ function createCoreStartMock() { return mock; } +function createInternalCorePrebootMock() { + const prebootDeps = { + context: contextServiceMock.createPrebootContract(), + elasticsearch: elasticsearchServiceMock.createInternalPreboot(), + http: httpServiceMock.createInternalPrebootContract(), + httpResources: httpResourcesMock.createPrebootContract(), + uiSettings: uiSettingsServiceMock.createPrebootContract(), + logging: loggingServiceMock.createInternalPrebootContract(), + preboot: prebootServiceMock.createInternalPrebootContract(), + }; + return prebootDeps; +} + function createInternalCoreSetupMock() { const setupDeps = { capabilities: capabilitiesServiceMock.createSetupContract(), @@ -227,8 +262,10 @@ function createCoreRequestHandlerContextMock() { } export const coreMock = { + createPreboot: createCorePrebootMock, createSetup: createCoreSetupMock, createStart: createCoreStartMock, + createInternalPreboot: createInternalCorePrebootMock, createInternalSetup: createInternalCoreSetupMock, createInternalStart: createInternalCoreStartMock, createPluginInitializerContext: pluginInitializerContextMock, diff --git a/src/core/server/plugins/discovery/plugin_manifest_parser.test.ts b/src/core/server/plugins/discovery/plugin_manifest_parser.test.ts index f3a92c896b0148..3e410e4b19c0ef 100644 --- a/src/core/server/plugins/discovery/plugin_manifest_parser.test.ts +++ b/src/core/server/plugins/discovery/plugin_manifest_parser.test.ts @@ -226,6 +226,29 @@ test('return error when manifest contains unrecognized properties', async () => }); }); +test('returns error when manifest contains unrecognized `type`', async () => { + mockReadFile.mockImplementation((path, cb) => { + cb( + null, + Buffer.from( + JSON.stringify({ + id: 'someId', + version: '7.0.0', + kibanaVersion: '7.0.0', + type: 'unknown', + server: true, + }) + ) + ); + }); + + await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({ + message: `The "type" in manifest for plugin "someId" is set to "unknown", but it should either be "standard" or "preboot". (invalid-manifest, ${pluginManifestPath})`, + type: PluginDiscoveryErrorType.InvalidManifest, + path: pluginManifestPath, + }); +}); + describe('configPath', () => { test('falls back to plugin id if not specified', async () => { mockReadFile.mockImplementation((path, cb) => { @@ -284,6 +307,7 @@ test('set defaults for all missing optional fields', async () => { configPath: 'some_id', version: '7.0.0', kibanaVersion: '7.0.0', + type: 'standard', optionalPlugins: [], requiredPlugins: [], requiredBundles: [], @@ -302,6 +326,7 @@ test('return all set optional fields as they are in manifest', async () => { configPath: ['some', 'path'], version: 'some-version', kibanaVersion: '7.0.0', + type: 'preboot', requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'], optionalPlugins: ['some-optional-plugin'], ui: true, @@ -315,6 +340,7 @@ test('return all set optional fields as they are in manifest', async () => { configPath: ['some', 'path'], version: 'some-version', kibanaVersion: '7.0.0', + type: 'preboot', optionalPlugins: ['some-optional-plugin'], requiredBundles: [], requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'], @@ -345,6 +371,7 @@ test('return manifest when plugin expected Kibana version matches actual version configPath: 'some-path', version: 'some-version', kibanaVersion: '7.0.0-alpha2', + type: 'standard', optionalPlugins: [], requiredPlugins: ['some-required-plugin'], requiredBundles: [], @@ -375,6 +402,7 @@ test('return manifest when plugin expected Kibana version is `kibana`', async () configPath: 'some_id', version: 'some-version', kibanaVersion: 'kibana', + type: 'standard', optionalPlugins: [], requiredPlugins: ['some-required-plugin'], requiredBundles: [], diff --git a/src/core/server/plugins/discovery/plugin_manifest_parser.ts b/src/core/server/plugins/discovery/plugin_manifest_parser.ts index b59418a67219e0..57640ec6acc0ac 100644 --- a/src/core/server/plugins/discovery/plugin_manifest_parser.ts +++ b/src/core/server/plugins/discovery/plugin_manifest_parser.ts @@ -12,7 +12,7 @@ import { coerce } from 'semver'; import { promisify } from 'util'; import { snakeCase } from 'lodash'; import { isConfigPath, PackageInfo } from '../../config'; -import { PluginManifest } from '../types'; +import { PluginManifest, PluginType } from '../types'; import { PluginDiscoveryError } from './plugin_discovery_error'; import { isCamelCase } from './is_camel_case'; @@ -39,6 +39,7 @@ const KNOWN_MANIFEST_FIELDS = (() => { const manifestFields: { [P in keyof PluginManifest]: boolean } = { id: true, kibanaVersion: true, + type: true, version: true, configPath: true, requiredPlugins: true, @@ -178,10 +179,21 @@ export async function parseManifest( ); } + const type = manifest.type ?? PluginType.standard; + if (type !== PluginType.preboot && type !== PluginType.standard) { + throw PluginDiscoveryError.invalidManifest( + manifestPath, + new Error( + `The "type" in manifest for plugin "${manifest.id}" is set to "${type}", but it should either be "standard" or "preboot".` + ) + ); + } + return { id: manifest.id, version: manifest.version, kibanaVersion: expectedKibanaVersion, + type, configPath: manifest.configPath || snakeCase(manifest.id), requiredPlugins: Array.isArray(manifest.requiredPlugins) ? manifest.requiredPlugins : [], optionalPlugins: Array.isArray(manifest.optionalPlugins) ? manifest.optionalPlugins : [], diff --git a/src/core/server/plugins/discovery/plugins_discovery.test.ts b/src/core/server/plugins/discovery/plugins_discovery.test.ts index f6028fa8b099d2..28f2ab799e0920 100644 --- a/src/core/server/plugins/discovery/plugins_discovery.test.ts +++ b/src/core/server/plugins/discovery/plugins_discovery.test.ts @@ -21,6 +21,7 @@ import { PluginsConfig, PluginsConfigType, config } from '../plugins_config'; import type { InstanceInfo } from '../plugin_context'; import { discover } from './plugins_discovery'; import { CoreContext } from '../../core_context'; +import { PluginType } from '../types'; const KIBANA_ROOT = process.cwd(); @@ -34,6 +35,15 @@ const Plugins = { incompatible: () => ({ 'kibana.json': JSON.stringify({ id: 'plugin', version: '1' }), }), + incompatibleType: (id: string) => ({ + 'kibana.json': JSON.stringify({ + id, + version: '1', + kibanaVersion: '1.2.3', + type: 'evenEarlierThanPreboot', + server: true, + }), + }), missingManifest: () => ({}), inaccessibleManifest: () => ({ 'kibana.json': mockFs.file({ @@ -52,6 +62,18 @@ const Plugins = { server: true, }), }), + validPreboot: (id: string) => ({ + 'kibana.json': JSON.stringify({ + id, + configPath: ['plugins', id], + version: '1', + kibanaVersion: '1.2.3', + type: PluginType.preboot, + requiredPlugins: [], + optionalPlugins: [], + server: true, + }), + }), }; const packageMock = { @@ -132,6 +154,7 @@ describe('plugins discovery system', () => { [`${KIBANA_ROOT}/src/plugins/plugin_a`]: Plugins.valid('pluginA'), [`${KIBANA_ROOT}/plugins/plugin_b`]: Plugins.valid('pluginB'), [`${KIBANA_ROOT}/x-pack/plugins/plugin_c`]: Plugins.valid('pluginC'), + [`${KIBANA_ROOT}/src/plugins/plugin_d`]: Plugins.validPreboot('pluginD'), }, { createCwd: false } ); @@ -139,8 +162,10 @@ describe('plugins discovery system', () => { const plugins = await plugin$.pipe(toArray()).toPromise(); const pluginNames = plugins.map((plugin) => plugin.name); - expect(pluginNames).toHaveLength(3); - expect(pluginNames).toEqual(expect.arrayContaining(['pluginA', 'pluginB', 'pluginC'])); + expect(pluginNames).toHaveLength(4); + expect(pluginNames).toEqual( + expect.arrayContaining(['pluginA', 'pluginB', 'pluginC', 'pluginD']) + ); }); it('return errors when the manifest is invalid or incompatible', async () => { @@ -155,6 +180,7 @@ describe('plugins discovery system', () => { [`${KIBANA_ROOT}/src/plugins/plugin_a`]: Plugins.invalid(), [`${KIBANA_ROOT}/src/plugins/plugin_b`]: Plugins.incomplete(), [`${KIBANA_ROOT}/src/plugins/plugin_c`]: Plugins.incompatible(), + [`${KIBANA_ROOT}/src/plugins/plugin_d`]: Plugins.incompatibleType('pluginD'), [`${KIBANA_ROOT}/src/plugins/plugin_ad`]: Plugins.missingManifest(), }, { createCwd: false } @@ -181,6 +207,9 @@ describe('plugins discovery system', () => { `Error: Plugin "plugin" is only compatible with Kibana version "1", but used Kibana version is "1.2.3". (incompatible-version, ${manifestPath( 'plugin_c' )})`, + `Error: The "type" in manifest for plugin "pluginD" is set to "evenEarlierThanPreboot", but it should either be "standard" or "preboot". (invalid-manifest, ${manifestPath( + 'plugin_d' + )})`, ]) ); }); @@ -271,7 +300,8 @@ describe('plugins discovery system', () => { [`${KIBANA_ROOT}/src/plugins/plugin_a`]: Plugins.valid('pluginA'), [`${KIBANA_ROOT}/src/plugins/sub1/plugin_b`]: Plugins.valid('pluginB'), [`${KIBANA_ROOT}/src/plugins/sub1/sub2/plugin_c`]: Plugins.valid('pluginC'), - [`${KIBANA_ROOT}/src/plugins/sub1/sub2/plugin_d`]: Plugins.incomplete(), + [`${KIBANA_ROOT}/src/plugins/sub1/sub2/plugin_d`]: Plugins.validPreboot('pluginD'), + [`${KIBANA_ROOT}/src/plugins/sub1/sub2/plugin_e`]: Plugins.incomplete(), }, { createCwd: false } ); @@ -279,8 +309,10 @@ describe('plugins discovery system', () => { const plugins = await plugin$.pipe(toArray()).toPromise(); const pluginNames = plugins.map((plugin) => plugin.name); - expect(pluginNames).toHaveLength(3); - expect(pluginNames).toEqual(expect.arrayContaining(['pluginA', 'pluginB', 'pluginC'])); + expect(pluginNames).toHaveLength(4); + expect(pluginNames).toEqual( + expect.arrayContaining(['pluginA', 'pluginB', 'pluginC', 'pluginD']) + ); const errors = await error$ .pipe( @@ -294,7 +326,7 @@ describe('plugins discovery system', () => { `Error: Plugin manifest must contain an "id" property. (invalid-manifest, ${manifestPath( 'sub1', 'sub2', - 'plugin_d' + 'plugin_e' )})`, ]) ); @@ -358,6 +390,7 @@ describe('plugins discovery system', () => { [pluginFolder]: { plugin_a: Plugins.valid('pluginA'), plugin_b: Plugins.valid('pluginB'), + plugin_c: Plugins.validPreboot('pluginC'), }, }, { createCwd: false } @@ -366,8 +399,8 @@ describe('plugins discovery system', () => { const plugins = await plugin$.pipe(toArray()).toPromise(); const pluginNames = plugins.map((plugin) => plugin.name); - expect(pluginNames).toHaveLength(2); - expect(pluginNames).toEqual(expect.arrayContaining(['pluginA', 'pluginB'])); + expect(pluginNames).toHaveLength(3); + expect(pluginNames).toEqual(expect.arrayContaining(['pluginA', 'pluginB', 'pluginC'])); }); it('logs a warning about --plugin-path when used in development', async () => { diff --git a/src/core/server/plugins/index.ts b/src/core/server/plugins/index.ts index a71df00b39c5cf..1b655ccd8bd98f 100644 --- a/src/core/server/plugins/index.ts +++ b/src/core/server/plugins/index.ts @@ -7,7 +7,12 @@ */ export { PluginsService } from './plugins_service'; -export type { PluginsServiceSetup, PluginsServiceStart, UiPlugins } from './plugins_service'; +export type { + PluginsServiceSetup, + PluginsServiceStart, + UiPlugins, + DiscoveredPlugins, +} from './plugins_service'; export { config } from './plugins_config'; /** @internal */ export { isNewPlatformPlugin } from './discovery'; diff --git a/src/core/server/plugins/integration_tests/plugins_service.test.ts b/src/core/server/plugins/integration_tests/plugins_service.test.ts index a29fb01fbc0092..1b0caf7bf6255c 100644 --- a/src/core/server/plugins/integration_tests/plugins_service.test.ts +++ b/src/core/server/plugins/integration_tests/plugins_service.test.ts @@ -20,12 +20,12 @@ import { config } from '../plugins_config'; import { loggingSystemMock } from '../../logging/logging_system.mock'; import { environmentServiceMock } from '../../environment/environment_service.mock'; import { coreMock } from '../../mocks'; -import { AsyncPlugin } from '../types'; +import { AsyncPlugin, PluginType } from '../types'; import { PluginWrapper } from '../plugin'; describe('PluginsService', () => { const logger = loggingSystemMock.create(); - const environmentSetup = environmentServiceMock.createSetupContract(); + const environmentPreboot = environmentServiceMock.createPrebootContract(); let pluginsService: PluginsService; const createPlugin = ( @@ -38,6 +38,7 @@ describe('PluginsService', () => { requiredBundles = [], optionalPlugins = [], kibanaVersion = '7.0.0', + type = PluginType.standard, configPath = [path], server = true, ui = true, @@ -49,6 +50,7 @@ describe('PluginsService', () => { requiredBundles?: string[]; optionalPlugins?: string[]; kibanaVersion?: string; + type?: PluginType; configPath?: ConfigPath; server?: boolean; ui?: boolean; @@ -61,6 +63,7 @@ describe('PluginsService', () => { version, configPath: `${configPath}${disabled ? '-disabled' : ''}`, kibanaVersion, + type, requiredPlugins, requiredBundles, optionalPlugins, @@ -150,7 +153,10 @@ describe('PluginsService', () => { } ); - await pluginsService.discover({ environment: environmentSetup }); + await pluginsService.discover({ environment: environmentPreboot }); + + const prebootDeps = coreMock.createInternalPreboot(); + await pluginsService.preboot(prebootDeps); const setupDeps = coreMock.createInternalSetup(); await pluginsService.setup(setupDeps); diff --git a/src/core/server/plugins/plugin.test.ts b/src/core/server/plugins/plugin.test.ts index c90d2e804225c7..610bc1cac5a3be 100644 --- a/src/core/server/plugins/plugin.test.ts +++ b/src/core/server/plugins/plugin.test.ts @@ -15,10 +15,10 @@ import { Env } from '../config'; import { CoreContext } from '../core_context'; import { coreMock } from '../mocks'; import { loggingSystemMock } from '../logging/logging_system.mock'; -import { getEnvOptions, configServiceMock } from '../config/mocks'; +import { configServiceMock, getEnvOptions } from '../config/mocks'; import { PluginWrapper } from './plugin'; -import { PluginManifest } from './types'; +import { PluginManifest, PluginType } from './types'; import { createPluginInitializerContext, createPluginSetupContext, @@ -45,6 +45,7 @@ function createPluginManifest(manifestProps: Partial = {}): Plug version: 'some-version', configPath: 'path', kibanaVersion: '7.0.0', + type: PluginType.standard, requiredPlugins: ['some-required-dep'], optionalPlugins: ['some-optional-dep'], requiredBundles: [], @@ -243,6 +244,31 @@ test('`start` fails if setup is not called first', () => { ); }); +test('`start` fails invoked for the `preboot` plugin', async () => { + const manifest = createPluginManifest({ type: PluginType.preboot }); + const opaqueId = Symbol(); + const plugin = new PluginWrapper({ + path: 'plugin-with-initializer-path', + manifest, + opaqueId, + initializerContext: createPluginInitializerContext( + coreContext, + opaqueId, + manifest, + instanceInfo + ), + }); + + const mockPluginInstance = { setup: jest.fn() }; + mockPluginInitializer.mockReturnValue(mockPluginInstance); + + await plugin.setup({} as any, {} as any); + + expect(() => plugin.start({} as any, {} as any)).toThrowErrorMatchingInlineSnapshot( + `"Plugin \\"some-plugin-id\\" is a preboot plugin and cannot be started."` + ); +}); + test('`start` calls plugin.start with context and dependencies', async () => { const manifest = createPluginManifest(); const opaqueId = Symbol(); diff --git a/src/core/server/plugins/plugin.ts b/src/core/server/plugins/plugin.ts index ca7f11e28de75f..76ed4f8f24b3d4 100644 --- a/src/core/server/plugins/plugin.ts +++ b/src/core/server/plugins/plugin.ts @@ -15,15 +15,17 @@ import { isConfigSchema } from '@kbn/config-schema'; import { Logger } from '../logging'; import { - Plugin, AsyncPlugin, + Plugin, + PluginConfigDescriptor, + PluginInitializer, PluginInitializerContext, PluginManifest, - PluginInitializer, PluginOpaqueId, - PluginConfigDescriptor, + PluginType, + PrebootPlugin, } from './types'; -import { CoreSetup, CoreStart } from '..'; +import { CorePreboot, CoreSetup, CoreStart } from '..'; /** * Lightweight wrapper around discovered plugin that is responsible for instantiating @@ -53,6 +55,7 @@ export class PluginWrapper< private instance?: | Plugin + | PrebootPlugin | AsyncPlugin; private readonly startDependencies$ = new Subject<[CoreStart, TPluginsStart, TStart]>(); @@ -88,11 +91,16 @@ export class PluginWrapper< * is the contract returned by the dependency's `setup` function. */ public setup( - setupContext: CoreSetup, + setupContext: CoreSetup | CorePreboot, plugins: TPluginsSetup ): TSetup | Promise { this.instance = this.createPluginInstance(); - return this.instance.setup(setupContext, plugins); + + if (this.isPrebootPluginInstance(this.instance)) { + return this.instance.setup(setupContext as CorePreboot, plugins); + } + + return this.instance.setup(setupContext as CoreSetup, plugins); } /** @@ -107,6 +115,10 @@ export class PluginWrapper< throw new Error(`Plugin "${this.name}" can't be started since it isn't set up.`); } + if (this.isPrebootPluginInstance(this.instance)) { + throw new Error(`Plugin "${this.name}" is a preboot plugin and cannot be started.`); + } + const startContract = this.instance.start(startContext, plugins); if (isPromise(startContract)) { return startContract.then((resolvedContract) => { @@ -185,4 +197,10 @@ export class PluginWrapper< return instance; } + + private isPrebootPluginInstance( + instance: PluginWrapper['instance'] + ): instance is PrebootPlugin { + return this.manifest.type === PluginType.preboot; + } } diff --git a/src/core/server/plugins/plugin_context.test.ts b/src/core/server/plugins/plugin_context.test.ts index 4ba34ccb2149f0..7913bad3cad17d 100644 --- a/src/core/server/plugins/plugin_context.test.ts +++ b/src/core/server/plugins/plugin_context.test.ts @@ -10,15 +10,21 @@ import { duration } from 'moment'; import { first } from 'rxjs/operators'; import { REPO_ROOT } from '@kbn/dev-utils'; import { fromRoot } from '@kbn/utils'; -import { createPluginInitializerContext, InstanceInfo } from './plugin_context'; +import { + createPluginInitializerContext, + createPluginPrebootSetupContext, + InstanceInfo, +} from './plugin_context'; import { CoreContext } from '../core_context'; import { Env } from '../config'; import { loggingSystemMock } from '../logging/logging_system.mock'; -import { rawConfigServiceMock, getEnvOptions } from '../config/mocks'; -import { PluginManifest } from './types'; +import { rawConfigServiceMock, getEnvOptions, configServiceMock } from '../config/mocks'; +import { PluginManifest, PluginType } from './types'; import { Server } from '../server'; import { schema, ByteSizeValue } from '@kbn/config-schema'; import { ConfigService } from '@kbn/config'; +import { PluginWrapper } from './plugin'; +import { coreMock } from '../mocks'; function createPluginManifest(manifestProps: Partial = {}): PluginManifest { return { @@ -26,6 +32,7 @@ function createPluginManifest(manifestProps: Partial = {}): Plug version: 'some-version', configPath: 'path', kibanaVersion: '7.0.0', + type: PluginType.standard, requiredPlugins: ['some-required-dep'], requiredBundles: [], optionalPlugins: ['some-optional-dep'], @@ -141,5 +148,67 @@ describe('createPluginInitializerContext', () => { ); expect(pluginInitializerContext.env.instanceUuid).toBe('kibana-uuid'); }); + + it('should expose paths to the config files', () => { + coreContext = { + ...coreContext, + env: Env.createDefault( + REPO_ROOT, + getEnvOptions({ + configs: ['/home/kibana/config/kibana.yml', '/home/kibana/config/kibana.dev.yml'], + }) + ), + }; + const pluginInitializerContext = createPluginInitializerContext( + coreContext, + opaqueId, + createPluginManifest(), + instanceInfo + ); + expect(pluginInitializerContext.env.configs).toEqual([ + '/home/kibana/config/kibana.yml', + '/home/kibana/config/kibana.dev.yml', + ]); + }); + }); +}); + +describe('createPluginPrebootSetupContext', () => { + let coreContext: CoreContext; + let opaqueId: symbol; + + beforeEach(async () => { + opaqueId = Symbol(); + coreContext = { + coreId: Symbol('core'), + env: Env.createDefault(REPO_ROOT, getEnvOptions()), + logger: loggingSystemMock.create(), + configService: configServiceMock.create(), + }; + }); + + it('`holdSetupUntilResolved` captures plugin.name', () => { + const manifest = createPluginManifest(); + const plugin = new PluginWrapper({ + path: 'some-path', + manifest, + opaqueId, + initializerContext: createPluginInitializerContext(coreContext, opaqueId, manifest, { + uuid: 'instance-uuid', + }), + }); + + const corePreboot = coreMock.createInternalPreboot(); + const prebootSetupContext = createPluginPrebootSetupContext(coreContext, corePreboot, plugin); + + const holdSetupPromise = Promise.resolve(undefined); + prebootSetupContext.preboot.holdSetupUntilResolved('some-reason', holdSetupPromise); + + expect(corePreboot.preboot.holdSetupUntilResolved).toHaveBeenCalledTimes(1); + expect(corePreboot.preboot.holdSetupUntilResolved).toHaveBeenCalledWith( + 'some-plugin-id', + 'some-reason', + holdSetupPromise + ); }); }); diff --git a/src/core/server/plugins/plugin_context.ts b/src/core/server/plugins/plugin_context.ts index 70fd1c60efa614..29194b1e8fc62b 100644 --- a/src/core/server/plugins/plugin_context.ts +++ b/src/core/server/plugins/plugin_context.ts @@ -10,11 +10,15 @@ import { shareReplay } from 'rxjs/operators'; import type { RequestHandlerContext } from 'src/core/server'; import { CoreContext } from '../core_context'; import { PluginWrapper } from './plugin'; -import { PluginsServiceSetupDeps, PluginsServiceStartDeps } from './plugins_service'; +import { + PluginsServicePrebootSetupDeps, + PluginsServiceSetupDeps, + PluginsServiceStartDeps, +} from './plugins_service'; import { PluginInitializerContext, PluginManifest, PluginOpaqueId } from './types'; import { IRouter, RequestHandlerContextProvider } from '../http'; import { getGlobalConfig, getGlobalConfig$ } from './legacy_config'; -import { CoreSetup, CoreStart } from '..'; +import { CorePreboot, CoreSetup, CoreStart } from '..'; export interface InstanceInfo { uuid: string; @@ -49,6 +53,7 @@ export function createPluginInitializerContext( mode: coreContext.env.mode, packageInfo: coreContext.env.packageInfo, instanceUuid: instanceInfo.uuid, + configs: coreContext.env.configs, }, /** @@ -83,6 +88,42 @@ export function createPluginInitializerContext( }; } +/** + * Provides `CorePreboot` contract that will be exposed to the `preboot` plugin `setup` method. + * This contract should be safe to use only within `setup` itself. + * + * This is called for each `preboot` plugin when it's set up, so each plugin gets its own + * version of these values. + * + * We should aim to be restrictive and specific in the APIs that we expose. + * + * @param coreContext Kibana core context + * @param deps Dependencies that Plugins services gets during setup. + * @param plugin The plugin we're building these values for. + * @internal + */ +export function createPluginPrebootSetupContext( + coreContext: CoreContext, + deps: PluginsServicePrebootSetupDeps, + plugin: PluginWrapper +): CorePreboot { + return { + elasticsearch: { + config: deps.elasticsearch.config, + createClient: deps.elasticsearch.createClient, + }, + http: { + registerRoutes: deps.http.registerRoutes, + basePath: deps.http.basePath, + }, + preboot: { + isSetupOnHold: deps.preboot.isSetupOnHold, + holdSetupUntilResolved: (reason, promise) => + deps.preboot.holdSetupUntilResolved(plugin.name, reason, promise), + }, + }; +} + /** * This returns a facade for `CoreContext` that will be exposed to the plugin `setup` method. * This facade should be safe to use only within `setup` itself. diff --git a/src/core/server/plugins/plugins_service.mock.ts b/src/core/server/plugins/plugins_service.mock.ts index f4f2263a1bdb06..ee7b35a412e800 100644 --- a/src/core/server/plugins/plugins_service.mock.ts +++ b/src/core/server/plugins/plugins_service.mock.ts @@ -20,6 +20,7 @@ const createStartContractMock = () => ({ contracts: new Map() }); const createServiceMock = (): PluginsServiceMock => ({ discover: jest.fn(), getExposedPluginConfigsToUsage: jest.fn(), + preboot: jest.fn(), setup: jest.fn().mockResolvedValue(createSetupContractMock()), start: jest.fn().mockResolvedValue(createStartContractMock()), stop: jest.fn(), diff --git a/src/core/server/plugins/plugins_service.test.ts b/src/core/server/plugins/plugins_service.test.ts index 5c50df07dc6979..3cd645cb74ac00 100644 --- a/src/core/server/plugins/plugins_service.test.ts +++ b/src/core/server/plugins/plugins_service.test.ts @@ -24,26 +24,31 @@ import { PluginsService } from './plugins_service'; import { PluginsSystem } from './plugins_system'; import { config } from './plugins_config'; import { take } from 'rxjs/operators'; -import { DiscoveredPlugin } from './types'; +import { DiscoveredPlugin, PluginType } from './types'; -const MockPluginsSystem: jest.Mock = PluginsSystem as any; +const MockPluginsSystem: jest.Mock> = PluginsSystem as any; let pluginsService: PluginsService; let config$: BehaviorSubject>; let configService: ConfigService; let coreId: symbol; let env: Env; -let mockPluginSystem: jest.Mocked; -let environmentSetup: ReturnType; +let prebootMockPluginSystem: jest.Mocked>; +let standardMockPluginSystem: jest.Mocked>; +let environmentPreboot: ReturnType; +const prebootDeps = coreMock.createInternalPreboot(); const setupDeps = coreMock.createInternalSetup(); +const startDeps = coreMock.createInternalStart(); const logger = loggingSystemMock.create(); expect.addSnapshotSerializer(createAbsolutePathSerializer()); ['path-1', 'path-2', 'path-3', 'path-4', 'path-5', 'path-6', 'path-7', 'path-8'].forEach((path) => { - jest.doMock(join(path, 'server'), () => ({}), { - virtual: true, + [PluginType.preboot, PluginType.standard].forEach((type) => { + jest.doMock(join(`${path}-${type}`, 'server'), () => ({}), { + virtual: true, + }); }); }); @@ -53,6 +58,7 @@ const createPlugin = ( path = id, disabled = false, version = 'some-version', + type = PluginType.standard, requiredPlugins = [], requiredBundles = [], optionalPlugins = [], @@ -64,6 +70,7 @@ const createPlugin = ( path?: string; disabled?: boolean; version?: string; + type?: PluginType; requiredPlugins?: string[]; requiredBundles?: string[]; optionalPlugins?: string[]; @@ -80,6 +87,7 @@ const createPlugin = ( version, configPath: disabled ? configPath.concat('-disabled') : configPath, kibanaVersion, + type, requiredPlugins, requiredBundles, optionalPlugins, @@ -111,11 +119,13 @@ async function testSetup() { await configService.setSchema(config.path, config.schema); pluginsService = new PluginsService({ coreId, env, logger, configService }); - [mockPluginSystem] = MockPluginsSystem.mock.instances as any; - mockPluginSystem.uiPlugins.mockReturnValue(new Map()); - mockPluginSystem.getPlugins.mockReturnValue([]); + [prebootMockPluginSystem, standardMockPluginSystem] = MockPluginsSystem.mock.instances as any; + prebootMockPluginSystem.uiPlugins.mockReturnValue(new Map()); + prebootMockPluginSystem.getPlugins.mockReturnValue([]); + standardMockPluginSystem.uiPlugins.mockReturnValue(new Map()); + standardMockPluginSystem.getPlugins.mockReturnValue([]); - environmentSetup = environmentServiceMock.createSetupContract(); + environmentPreboot = environmentServiceMock.createPrebootContract(); } afterEach(() => { @@ -134,7 +144,7 @@ describe('PluginsService', () => { plugin$: from([]), }); - await expect(pluginsService.discover({ environment: environmentSetup })).rejects + await expect(pluginsService.discover({ environment: environmentPreboot })).rejects .toMatchInlineSnapshot(` [Error: Failed to initialize plugins: Invalid JSON (invalid-manifest, path-1)] @@ -156,7 +166,7 @@ describe('PluginsService', () => { plugin$: from([]), }); - await expect(pluginsService.discover({ environment: environmentSetup })).rejects + await expect(pluginsService.discover({ environment: environmentPreboot })).rejects .toMatchInlineSnapshot(` [Error: Failed to initialize plugins: Incompatible version (incompatible-version, path-3)] @@ -175,14 +185,14 @@ describe('PluginsService', () => { error$: from([]), plugin$: from([ createPlugin('conflicting-id', { - path: 'path-4', + path: 'path-4-standard', version: 'some-version', configPath: 'path', requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'], optionalPlugins: ['some-optional-plugin'], }), createPlugin('conflicting-id', { - path: 'path-4', + path: 'path-4-standard', version: 'some-version', configPath: 'path', requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'], @@ -192,13 +202,49 @@ describe('PluginsService', () => { }); await expect( - pluginsService.discover({ environment: environmentSetup }) + pluginsService.discover({ environment: environmentPreboot }) ).rejects.toMatchInlineSnapshot( `[Error: Plugin with id "conflicting-id" is already registered!]` ); - expect(mockPluginSystem.addPlugin).not.toHaveBeenCalled(); - expect(mockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + expect(prebootMockPluginSystem.addPlugin).not.toHaveBeenCalled(); + expect(prebootMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.addPlugin).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + }); + + it('throws if discovered standard and preboot plugins with conflicting names', async () => { + mockDiscover.mockReturnValue({ + error$: from([]), + plugin$: from([ + createPlugin('conflicting-id', { + type: PluginType.preboot, + path: 'path-4-preboot', + version: 'some-version', + configPath: 'path', + requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'], + optionalPlugins: ['some-optional-plugin'], + }), + createPlugin('conflicting-id', { + path: 'path-4-standard', + version: 'some-version', + configPath: 'path', + requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'], + optionalPlugins: ['some-optional-plugin'], + }), + ]), + }); + + await expect( + pluginsService.discover({ environment: environmentPreboot }) + ).rejects.toMatchInlineSnapshot( + `[Error: Plugin with id "conflicting-id" is already registered!]` + ); + + expect(prebootMockPluginSystem.addPlugin).not.toHaveBeenCalled(); + expect(prebootMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.addPlugin).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); }); it('properly detects plugins that should be disabled.', async () => { @@ -206,160 +252,356 @@ describe('PluginsService', () => { .spyOn(configService, 'isEnabledAtPath') .mockImplementation((path) => Promise.resolve(!path.includes('disabled'))); - mockPluginSystem.setupPlugins.mockResolvedValue(new Map()); + prebootMockPluginSystem.setupPlugins.mockResolvedValue(new Map()); + standardMockPluginSystem.setupPlugins.mockResolvedValue(new Map()); mockDiscover.mockReturnValue({ error$: from([]), plugin$: from([ - createPlugin('explicitly-disabled-plugin', { + createPlugin('explicitly-disabled-plugin-preboot', { + type: PluginType.preboot, + disabled: true, + path: 'path-1-preboot', + configPath: 'path-1-preboot', + }), + createPlugin('explicitly-disabled-plugin-standard', { disabled: true, - path: 'path-1', - configPath: 'path-1', + path: 'path-1-standard', + configPath: 'path-1-standard', + }), + createPlugin('plugin-with-missing-required-deps-preboot', { + type: PluginType.preboot, + path: 'path-2-preboot', + configPath: 'path-2-preboot', + requiredPlugins: ['missing-plugin-preboot'], }), - createPlugin('plugin-with-missing-required-deps', { - path: 'path-2', - configPath: 'path-2', - requiredPlugins: ['missing-plugin'], + createPlugin('plugin-with-missing-required-deps-standard', { + path: 'path-2-standard', + configPath: 'path-2-standard', + requiredPlugins: ['missing-plugin-standard'], }), - createPlugin('plugin-with-disabled-transitive-dep', { - path: 'path-3', - configPath: 'path-3', - requiredPlugins: ['another-explicitly-disabled-plugin'], + createPlugin('plugin-with-disabled-transitive-dep-preboot', { + type: PluginType.preboot, + path: 'path-3-preboot', + configPath: 'path-3-preboot', + requiredPlugins: ['another-explicitly-disabled-plugin-preboot'], }), - createPlugin('another-explicitly-disabled-plugin', { + createPlugin('plugin-with-disabled-transitive-dep-standard', { + path: 'path-3-standard', + configPath: 'path-3-standard', + requiredPlugins: ['another-explicitly-disabled-plugin-standard'], + }), + createPlugin('another-explicitly-disabled-plugin-preboot', { + type: PluginType.preboot, + disabled: true, + path: 'path-4-preboot', + configPath: 'path-4-disabled-preboot', + }), + createPlugin('another-explicitly-disabled-plugin-standard', { disabled: true, - path: 'path-4', - configPath: 'path-4-disabled', + path: 'path-4-standard', + configPath: 'path-4-disabled-standard', + }), + createPlugin('plugin-with-disabled-optional-dep-preboot', { + type: PluginType.preboot, + path: 'path-5-preboot', + configPath: 'path-5-preboot', + optionalPlugins: ['explicitly-disabled-plugin-preboot'], }), - createPlugin('plugin-with-disabled-optional-dep', { - path: 'path-5', - configPath: 'path-5', - optionalPlugins: ['explicitly-disabled-plugin'], + createPlugin('plugin-with-disabled-optional-dep-standard', { + path: 'path-5-standard', + configPath: 'path-5-standard', + optionalPlugins: ['explicitly-disabled-plugin-standard'], }), - createPlugin('plugin-with-missing-optional-dep', { - path: 'path-6', - configPath: 'path-6', - optionalPlugins: ['missing-plugin'], + createPlugin('plugin-with-missing-optional-dep-preboot', { + type: PluginType.preboot, + path: 'path-6-preboot', + configPath: 'path-6-preboot', + optionalPlugins: ['missing-plugin-preboot'], }), - createPlugin('plugin-with-disabled-nested-transitive-dep', { - path: 'path-7', - configPath: 'path-7', - requiredPlugins: ['plugin-with-disabled-transitive-dep'], + createPlugin('plugin-with-missing-optional-dep-standard', { + path: 'path-6-standard', + configPath: 'path-6-standard', + optionalPlugins: ['missing-plugin-standard'], }), - createPlugin('plugin-with-missing-nested-dep', { - path: 'path-8', - configPath: 'path-8', - requiredPlugins: ['plugin-with-missing-required-deps'], + createPlugin('plugin-with-disabled-nested-transitive-dep-preboot', { + type: PluginType.preboot, + path: 'path-7-preboot', + configPath: 'path-7-preboot', + requiredPlugins: ['plugin-with-disabled-transitive-dep-preboot'], + }), + createPlugin('plugin-with-disabled-nested-transitive-dep-standard', { + path: 'path-7-standard', + configPath: 'path-7-standard', + requiredPlugins: ['plugin-with-disabled-transitive-dep-standard'], + }), + createPlugin('plugin-with-missing-nested-dep-preboot', { + type: PluginType.preboot, + path: 'path-8-preboot', + configPath: 'path-8-preboot', + requiredPlugins: ['plugin-with-missing-required-deps-preboot'], + }), + createPlugin('plugin-with-missing-nested-dep-standard', { + path: 'path-8-standard', + configPath: 'path-8-standard', + requiredPlugins: ['plugin-with-missing-required-deps-standard'], }), ]), }); - await pluginsService.discover({ environment: environmentSetup }); + await pluginsService.discover({ environment: environmentPreboot }); + await pluginsService.preboot(prebootDeps); const setup = await pluginsService.setup(setupDeps); expect(setup.contracts).toBeInstanceOf(Map); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); - expect(mockPluginSystem.setupPlugins).toHaveBeenCalledTimes(1); - expect(mockPluginSystem.setupPlugins).toHaveBeenCalledWith(setupDeps); + + expect(prebootMockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); + expect(standardMockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); + + expect(prebootMockPluginSystem.setupPlugins).toHaveBeenCalledTimes(1); + expect(standardMockPluginSystem.setupPlugins).toHaveBeenCalledTimes(1); + + expect(prebootMockPluginSystem.setupPlugins).toHaveBeenCalledWith(prebootDeps); + expect(standardMockPluginSystem.setupPlugins).toHaveBeenCalledWith(setupDeps); expect(loggingSystemMock.collect(logger).info).toMatchInlineSnapshot(` Array [ Array [ - "Plugin \\"explicitly-disabled-plugin\\" is disabled.", + "Plugin \\"explicitly-disabled-plugin-preboot\\" is disabled.", + ], + Array [ + "Plugin \\"explicitly-disabled-plugin-standard\\" is disabled.", + ], + Array [ + "Plugin \\"plugin-with-missing-required-deps-preboot\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [missing-plugin-preboot]", + ], + Array [ + "Plugin \\"plugin-with-missing-required-deps-standard\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [missing-plugin-standard]", + ], + Array [ + "Plugin \\"plugin-with-disabled-transitive-dep-preboot\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [another-explicitly-disabled-plugin-preboot]", + ], + Array [ + "Plugin \\"plugin-with-disabled-transitive-dep-standard\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [another-explicitly-disabled-plugin-standard]", ], Array [ - "Plugin \\"plugin-with-missing-required-deps\\" has been disabled since the following direct or transitive dependencies are missing or disabled: [missing-plugin]", + "Plugin \\"another-explicitly-disabled-plugin-preboot\\" is disabled.", ], Array [ - "Plugin \\"plugin-with-disabled-transitive-dep\\" has been disabled since the following direct or transitive dependencies are missing or disabled: [another-explicitly-disabled-plugin]", + "Plugin \\"another-explicitly-disabled-plugin-standard\\" is disabled.", ], Array [ - "Plugin \\"another-explicitly-disabled-plugin\\" is disabled.", + "Plugin \\"plugin-with-disabled-nested-transitive-dep-preboot\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [plugin-with-disabled-transitive-dep-preboot]", ], Array [ - "Plugin \\"plugin-with-disabled-nested-transitive-dep\\" has been disabled since the following direct or transitive dependencies are missing or disabled: [plugin-with-disabled-transitive-dep]", + "Plugin \\"plugin-with-disabled-nested-transitive-dep-standard\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [plugin-with-disabled-transitive-dep-standard]", ], Array [ - "Plugin \\"plugin-with-missing-nested-dep\\" has been disabled since the following direct or transitive dependencies are missing or disabled: [plugin-with-missing-required-deps]", + "Plugin \\"plugin-with-missing-nested-dep-preboot\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [plugin-with-missing-required-deps-preboot]", + ], + Array [ + "Plugin \\"plugin-with-missing-nested-dep-standard\\" has been disabled since the following direct or transitive dependencies are missing, disabled, or have incompatible types: [plugin-with-missing-required-deps-standard]", ], ] `); }); it('does not throw in case of mutual plugin dependencies', async () => { - const firstPlugin = createPlugin('first-plugin', { - path: 'path-1', - requiredPlugins: ['second-plugin'], + const prebootPlugins = [ + createPlugin('first-plugin-preboot', { + type: PluginType.preboot, + path: 'path-1-preboot', + requiredPlugins: ['second-plugin-preboot'], + }), + createPlugin('second-plugin-preboot', { + type: PluginType.preboot, + path: 'path-2-preboot', + requiredPlugins: ['first-plugin-preboot'], + }), + ]; + const standardPlugins = [ + createPlugin('first-plugin-standard', { + path: 'path-1-standard', + requiredPlugins: ['second-plugin-standard'], + }), + createPlugin('second-plugin-standard', { + path: 'path-2-standard', + requiredPlugins: ['first-plugin-standard'], + }), + ]; + + mockDiscover.mockReturnValue({ + error$: from([]), + plugin$: from([...prebootPlugins, ...standardPlugins]), }); - const secondPlugin = createPlugin('second-plugin', { - path: 'path-2', - requiredPlugins: ['first-plugin'], + + const { preboot, standard } = await pluginsService.discover({ + environment: environmentPreboot, }); + expect(mockDiscover).toHaveBeenCalledTimes(1); + + expect(preboot.pluginTree).toBeUndefined(); + for (const plugin of prebootPlugins) { + expect(prebootMockPluginSystem.addPlugin).toHaveBeenCalledWith(plugin); + } + expect(standard.pluginTree).toBeUndefined(); + for (const plugin of standardPlugins) { + expect(standardMockPluginSystem.addPlugin).toHaveBeenCalledWith(plugin); + } + }); + + it('does not throw in case of mutual plugin dependencies between preboot and standard plugins', async () => { mockDiscover.mockReturnValue({ error$: from([]), - plugin$: from([firstPlugin, secondPlugin]), + plugin$: from([ + createPlugin('first-plugin-preboot', { + type: PluginType.preboot, + path: 'path-1-preboot', + requiredPlugins: ['second-plugin-standard'], + }), + createPlugin('first-plugin-standard', { + path: 'path-1-standard', + requiredPlugins: ['second-plugin-preboot'], + }), + createPlugin('second-plugin-preboot', { + type: PluginType.preboot, + path: 'path-2-preboot', + requiredPlugins: ['first-plugin-standard'], + }), + createPlugin('second-plugin-standard', { + path: 'path-2-standard', + requiredPlugins: ['first-plugin-preboot'], + }), + ]), }); - const { pluginTree } = await pluginsService.discover({ environment: environmentSetup }); - expect(pluginTree).toBeUndefined(); + const { preboot, standard } = await pluginsService.discover({ + environment: environmentPreboot, + }); + expect(preboot.pluginTree).toBeUndefined(); + expect(standard.pluginTree).toBeUndefined(); expect(mockDiscover).toHaveBeenCalledTimes(1); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(firstPlugin); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(secondPlugin); + expect(prebootMockPluginSystem.addPlugin).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.addPlugin).not.toHaveBeenCalled(); }); it('does not throw in case of cyclic plugin dependencies', async () => { - const firstPlugin = createPlugin('first-plugin', { - path: 'path-1', - requiredPlugins: ['second-plugin'], - }); - const secondPlugin = createPlugin('second-plugin', { - path: 'path-2', - requiredPlugins: ['third-plugin', 'last-plugin'], - }); - const thirdPlugin = createPlugin('third-plugin', { - path: 'path-3', - requiredPlugins: ['last-plugin', 'first-plugin'], - }); - const lastPlugin = createPlugin('last-plugin', { - path: 'path-4', - requiredPlugins: ['first-plugin'], - }); - const missingDepsPlugin = createPlugin('missing-deps-plugin', { - path: 'path-5', - requiredPlugins: ['not-a-plugin'], - }); + const prebootPlugins = [ + createPlugin('first-plugin-preboot', { + type: PluginType.preboot, + path: 'path-1-preboot', + requiredPlugins: ['second-plugin-preboot'], + }), + createPlugin('second-plugin-preboot', { + type: PluginType.preboot, + path: 'path-2-preboot', + requiredPlugins: ['third-plugin-preboot', 'last-plugin-preboot'], + }), + createPlugin('third-plugin-preboot', { + type: PluginType.preboot, + path: 'path-3-preboot', + requiredPlugins: ['last-plugin-preboot', 'first-plugin-preboot'], + }), + createPlugin('last-plugin-preboot', { + type: PluginType.preboot, + path: 'path-4-preboot', + requiredPlugins: ['first-plugin-preboot'], + }), + createPlugin('missing-deps-plugin-preboot', { + type: PluginType.preboot, + path: 'path-5-preboot', + requiredPlugins: ['not-a-plugin-preboot'], + }), + ]; + + const standardPlugins = [ + createPlugin('first-plugin-standard', { + path: 'path-1-standard', + requiredPlugins: ['second-plugin-standard'], + }), + createPlugin('second-plugin-standard', { + path: 'path-2-standard', + requiredPlugins: ['third-plugin-standard', 'last-plugin-standard'], + }), + createPlugin('third-plugin-standard', { + path: 'path-3-standard', + requiredPlugins: ['last-plugin-standard', 'first-plugin-standard'], + }), + createPlugin('last-plugin-standard', { + path: 'path-4-standard', + requiredPlugins: ['first-plugin-standard'], + }), + createPlugin('missing-deps-plugin-standard', { + path: 'path-5-standard', + requiredPlugins: ['not-a-plugin-standard'], + }), + ]; mockDiscover.mockReturnValue({ error$: from([]), - plugin$: from([firstPlugin, secondPlugin, thirdPlugin, lastPlugin, missingDepsPlugin]), + plugin$: from([...prebootPlugins, ...standardPlugins]), }); - const { pluginTree } = await pluginsService.discover({ environment: environmentSetup }); - expect(pluginTree).toBeUndefined(); - + const { standard, preboot } = await pluginsService.discover({ + environment: environmentPreboot, + }); expect(mockDiscover).toHaveBeenCalledTimes(1); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledTimes(4); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(firstPlugin); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(secondPlugin); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(thirdPlugin); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(lastPlugin); + + expect(preboot.pluginTree).toBeUndefined(); + expect(prebootMockPluginSystem.addPlugin).toHaveBeenCalledTimes(4); + for (const plugin of prebootPlugins) { + if (plugin.name.startsWith('missing-deps')) { + expect(prebootMockPluginSystem.addPlugin).not.toHaveBeenCalledWith(plugin); + } else { + expect(prebootMockPluginSystem.addPlugin).toHaveBeenCalledWith(plugin); + } + } + + expect(standard.pluginTree).toBeUndefined(); + expect(standardMockPluginSystem.addPlugin).toHaveBeenCalledTimes(4); + for (const plugin of standardPlugins) { + if (plugin.name.startsWith('missing-deps')) { + expect(standardMockPluginSystem.addPlugin).not.toHaveBeenCalledWith(plugin); + } else { + expect(standardMockPluginSystem.addPlugin).toHaveBeenCalledWith(plugin); + } + } }); it('properly invokes plugin discovery and ignores non-critical errors.', async () => { - const firstPlugin = createPlugin('some-id', { - path: 'path-1', - configPath: 'path', - requiredPlugins: ['some-other-id'], - optionalPlugins: ['missing-optional-dep'], - }); - const secondPlugin = createPlugin('some-other-id', { - path: 'path-2', - version: 'some-other-version', - configPath: ['plugin', 'path'], - }); + const prebootPlugins = [ + createPlugin('some-id-preboot', { + type: PluginType.preboot, + path: 'path-1-preboot', + configPath: 'path-preboot', + requiredPlugins: ['some-other-id-preboot'], + optionalPlugins: ['missing-optional-dep'], + }), + createPlugin('some-other-id-preboot', { + type: PluginType.preboot, + path: 'path-2-preboot', + version: 'some-other-version', + configPath: ['plugin-other-preboot', 'path'], + }), + ]; + + const standardPlugins = [ + createPlugin('some-id-standard', { + type: PluginType.standard, + path: 'path-1-standard', + configPath: 'path-standard', + requiredPlugins: ['some-other-id-standard'], + optionalPlugins: ['missing-optional-dep'], + }), + createPlugin('some-other-id-standard', { + type: PluginType.standard, + path: 'path-2-standard', + version: 'some-other-version', + configPath: ['plugin-other-standard', 'path'], + }), + ]; mockDiscover.mockReturnValue({ error$: from([ @@ -367,13 +609,20 @@ describe('PluginsService', () => { PluginDiscoveryError.invalidSearchPath('dir-1', new Error('No dir')), PluginDiscoveryError.invalidPluginPath('path4-1', new Error('No path')), ]), - plugin$: from([firstPlugin, secondPlugin]), + plugin$: from([...prebootPlugins, ...standardPlugins]), }); - await pluginsService.discover({ environment: environmentSetup }); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(firstPlugin); - expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(secondPlugin); + await pluginsService.discover({ environment: environmentPreboot }); + expect(prebootMockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); + for (const plugin of prebootPlugins) { + expect(prebootMockPluginSystem.addPlugin).toHaveBeenCalledWith(plugin); + } + + expect(standardMockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); + for (const plugin of standardPlugins) { + expect(standardMockPluginSystem.addPlugin).toHaveBeenCalledWith(plugin); + } + expect(mockDiscover).toHaveBeenCalledTimes(1); expect(mockDiscover).toHaveBeenCalledWith( { @@ -399,27 +648,33 @@ describe('PluginsService', () => { const configSchema = schema.string(); jest.spyOn(configService, 'setSchema').mockImplementation(() => Promise.resolve()); jest.doMock( - join('path-with-schema', 'server'), - () => ({ - config: { - schema: configSchema, - }, - }), - { - virtual: true, - } + join('path-with-schema-preboot', 'server'), + () => ({ config: { schema: configSchema } }), + { virtual: true } + ); + jest.doMock( + join('path-with-schema-standard', 'server'), + () => ({ config: { schema: configSchema } }), + { virtual: true } ); + mockDiscover.mockReturnValue({ error$: from([]), plugin$: from([ - createPlugin('some-id', { - path: 'path-with-schema', - configPath: 'path', + createPlugin('some-id-preboot', { + type: PluginType.preboot, + path: 'path-with-schema-preboot', + configPath: 'path-preboot', + }), + createPlugin('some-id-standard', { + path: 'path-with-schema-standard', + configPath: 'path-standard', }), ]), }); - await pluginsService.discover({ environment: environmentSetup }); - expect(configService.setSchema).toBeCalledWith('path', configSchema); + await pluginsService.discover({ environment: environmentPreboot }); + expect(configService.setSchema).toBeCalledWith('path-preboot', configSchema); + expect(configService.setSchema).toBeCalledWith('path-standard', configSchema); }); it('registers plugin config deprecation provider in config service', async () => { @@ -427,127 +682,183 @@ describe('PluginsService', () => { jest.spyOn(configService, 'setSchema').mockImplementation(() => Promise.resolve()); jest.spyOn(configService, 'addDeprecationProvider'); - const deprecationProvider = () => []; + const prebootDeprecationProvider = () => []; jest.doMock( - join('path-with-provider', 'server'), - () => ({ - config: { - schema: configSchema, - deprecations: deprecationProvider, - }, - }), - { - virtual: true, - } + join('path-with-provider-preboot', 'server'), + () => ({ config: { schema: configSchema, deprecations: prebootDeprecationProvider } }), + { virtual: true } + ); + + const standardDeprecationProvider = () => []; + jest.doMock( + join('path-with-provider-standard', 'server'), + () => ({ config: { schema: configSchema, deprecations: standardDeprecationProvider } }), + { virtual: true } ); + mockDiscover.mockReturnValue({ error$: from([]), plugin$: from([ - createPlugin('some-id', { - path: 'path-with-provider', - configPath: 'config-path', + createPlugin('some-id-preboot', { + type: PluginType.preboot, + path: 'path-with-provider-preboot', + configPath: 'config-path-preboot', + }), + createPlugin('some-id-standard', { + path: 'path-with-provider-standard', + configPath: 'config-path-standard', }), ]), }); - await pluginsService.discover({ environment: environmentSetup }); + await pluginsService.discover({ environment: environmentPreboot }); + expect(configService.addDeprecationProvider).toBeCalledWith( + 'config-path-preboot', + prebootDeprecationProvider + ); expect(configService.addDeprecationProvider).toBeCalledWith( - 'config-path', - deprecationProvider + 'config-path-standard', + standardDeprecationProvider ); }); it('returns the paths of the plugins', async () => { - const pluginA = createPlugin('A', { path: '/plugin-A-path', configPath: 'pathA' }); - const pluginB = createPlugin('B', { path: '/plugin-B-path', configPath: 'pathB' }); - mockDiscover.mockReturnValue({ error$: from([]), plugin$: from([]), }); - mockPluginSystem.getPlugins.mockReturnValue([pluginA, pluginB]); - - const { pluginPaths } = await pluginsService.discover({ environment: environmentSetup }); + prebootMockPluginSystem.getPlugins.mockImplementation(() => [ + createPlugin('A-preboot', { + type: PluginType.preboot, + path: '/plugin-A-path-preboot', + configPath: 'pathA-preboot', + }), + createPlugin('B-preboot', { + type: PluginType.preboot, + path: '/plugin-B-path-preboot', + configPath: 'pathB-preboot', + }), + ]); - expect(pluginPaths).toEqual(['/plugin-A-path', '/plugin-B-path']); - }); + standardMockPluginSystem.getPlugins.mockImplementation(() => [ + createPlugin('A-standard', { + path: '/plugin-A-path-standard', + configPath: 'pathA-standard', + }), + createPlugin('B-standard', { + path: '/plugin-B-path-standard', + configPath: 'pathB-standard', + }), + ]); - it('ppopulates pluginConfigUsageDescriptors with plugins exposeToUsage property', async () => { - const pluginA = createPlugin('plugin-with-expose-usage', { - path: 'plugin-with-expose-usage', - configPath: 'pathA', + const { preboot, standard } = await pluginsService.discover({ + environment: environmentPreboot, }); - jest.doMock( - join('plugin-with-expose-usage', 'server'), - () => ({ - config: { - exposeToUsage: { - test: true, - nested: { - prop: true, - }, + expect(preboot.pluginPaths).toEqual(['/plugin-A-path-preboot', '/plugin-B-path-preboot']); + expect(standard.pluginPaths).toEqual(['/plugin-A-path-standard', '/plugin-B-path-standard']); + }); + + it('populates pluginConfigUsageDescriptors with plugins exposeToUsage property', async () => { + const pluginsWithExposeUsage = [ + createPlugin('plugin-with-expose-usage-preboot', { + type: PluginType.preboot, + path: 'plugin-with-expose-usage-preboot', + configPath: 'pathA-preboot', + }), + createPlugin('plugin-with-expose-usage-standard', { + path: 'plugin-with-expose-usage-standard', + configPath: 'pathA-standard', + }), + ]; + for (const plugin of pluginsWithExposeUsage) { + jest.doMock( + join(plugin.path, 'server'), + () => ({ + config: { + exposeToUsage: { test: true, nested: { prop: true } }, + schema: schema.maybe(schema.any()), }, - schema: schema.maybe(schema.any()), - }, + }), + { virtual: true } + ); + } + + const pluginsWithArrayConfigPath = [ + createPlugin('plugin-with-array-configPath-preboot', { + type: PluginType.preboot, + path: 'plugin-with-array-configPath-preboot', + version: 'some-other-version', + configPath: ['plugin-preboot', 'pathB'], }), - { - virtual: true, - } - ); - - const pluginB = createPlugin('plugin-with-array-configPath', { - path: 'plugin-with-array-configPath', - configPath: ['plugin', 'pathB'], - }); - - jest.doMock( - join('plugin-with-array-configPath', 'server'), - () => ({ - config: { - exposeToUsage: { - test: true, + createPlugin('plugin-with-array-configPath-standard', { + path: 'plugin-with-array-configPath-standard', + version: 'some-other-version', + configPath: ['plugin-standard', 'pathB'], + }), + ]; + for (const plugin of pluginsWithArrayConfigPath) { + jest.doMock( + join(plugin.path, 'server'), + () => ({ + config: { + exposeToUsage: { test: true }, + schema: schema.maybe(schema.any()), }, - schema: schema.maybe(schema.any()), - }, + }), + { virtual: true } + ); + } + + const pluginsWithoutExpose = [ + createPlugin('plugin-without-expose-preboot', { + type: PluginType.preboot, + path: 'plugin-without-expose-preboot', + configPath: 'pathC-preboot', }), - { - virtual: true, - } - ); - - jest.doMock( - join('plugin-without-expose', 'server'), - () => ({ - config: { - schema: schema.maybe(schema.any()), - }, + createPlugin('plugin-without-expose-standard', { + path: 'plugin-without-expose-standard', + configPath: 'pathC-standard', }), - { - virtual: true, - } - ); - - const pluginC = createPlugin('plugin-without-expose', { - path: 'plugin-without-expose', - configPath: 'pathC', - }); + ]; + for (const plugin of pluginsWithoutExpose) { + jest.doMock( + join(plugin.path, 'server'), + () => ({ + config: { + schema: schema.maybe(schema.any()), + }, + }), + { virtual: true } + ); + } mockDiscover.mockReturnValue({ error$: from([]), - plugin$: from([pluginA, pluginB, pluginC]), + plugin$: from([ + ...pluginsWithExposeUsage, + ...pluginsWithArrayConfigPath, + ...pluginsWithoutExpose, + ]), }); - await pluginsService.discover({ environment: environmentSetup }); + await pluginsService.discover({ environment: environmentPreboot }); // eslint-disable-next-line dot-notation expect(pluginsService['pluginConfigUsageDescriptors']).toMatchInlineSnapshot(` Map { - "pathA" => Object { + "pathA-preboot" => Object { "nested.prop": true, "test": true, }, - "plugin.pathB" => Object { + "pathA-standard" => Object { + "nested.prop": true, + "test": true, + }, + "plugin-preboot.pathB" => Object { + "test": true, + }, + "plugin-standard.pathB" => Object { "test": true, }, } @@ -560,6 +871,7 @@ describe('PluginsService', () => { plugin.name, { id: plugin.name, + type: plugin.manifest.type, configPath: plugin.manifest.configPath, requiredPlugins: [], requiredBundles: [], @@ -568,140 +880,246 @@ describe('PluginsService', () => { ]; it('properly generates client configs for plugins according to `exposeToBrowser`', async () => { - jest.doMock( - join('plugin-with-expose', 'server'), - () => ({ - config: { - exposeToBrowser: { - sharedProp: true, - }, - schema: schema.object({ - serverProp: schema.string({ defaultValue: 'serverProp default value' }), - sharedProp: schema.string({ defaultValue: 'sharedProp default value' }), - }), - }, - }), - { - virtual: true, - } - ); - const plugin = createPlugin('plugin-with-expose', { - path: 'plugin-with-expose', - configPath: 'path', + const prebootPlugin = createPlugin('plugin-with-expose-preboot', { + type: PluginType.preboot, + path: 'plugin-with-expose-preboot', + configPath: 'path-preboot', }); + const standardPlugin = createPlugin('plugin-with-expose-standard', { + path: 'plugin-with-expose-standard', + configPath: 'path-standard', + }); + for (const plugin of [prebootPlugin, standardPlugin]) { + jest.doMock( + join(plugin.path, 'server'), + () => ({ + config: { + exposeToBrowser: { + sharedProp: true, + }, + schema: schema.object({ + serverProp: schema.string({ + defaultValue: `serverProp default value ${plugin.name}`, + }), + sharedProp: schema.string({ + defaultValue: `sharedProp default value ${plugin.name}`, + }), + }), + }, + }), + { virtual: true } + ); + } + mockDiscover.mockReturnValue({ error$: from([]), - plugin$: from([plugin]), + plugin$: from([prebootPlugin, standardPlugin]), + }); + prebootMockPluginSystem.uiPlugins.mockReturnValue( + new Map([pluginToDiscoveredEntry(prebootPlugin)]) + ); + standardMockPluginSystem.uiPlugins.mockReturnValue( + new Map([pluginToDiscoveredEntry(standardPlugin)]) + ); + + const { preboot, standard } = await pluginsService.discover({ + environment: environmentPreboot, }); - mockPluginSystem.uiPlugins.mockReturnValue(new Map([pluginToDiscoveredEntry(plugin)])); - const { uiPlugins } = await pluginsService.discover({ environment: environmentSetup }); - const uiConfig$ = uiPlugins.browserConfigs.get('plugin-with-expose'); - expect(uiConfig$).toBeDefined(); + const prebootUIConfig$ = preboot.uiPlugins.browserConfigs.get('plugin-with-expose-preboot')!; + await expect(prebootUIConfig$.pipe(take(1)).toPromise()).resolves.toEqual({ + sharedProp: 'sharedProp default value plugin-with-expose-preboot', + }); - const uiConfig = await uiConfig$!.pipe(take(1)).toPromise(); - expect(uiConfig).toMatchInlineSnapshot(` - Object { - "sharedProp": "sharedProp default value", - } - `); + const standardUIConfig$ = standard.uiPlugins.browserConfigs.get( + 'plugin-with-expose-standard' + )!; + await expect(standardUIConfig$.pipe(take(1)).toPromise()).resolves.toEqual({ + sharedProp: 'sharedProp default value plugin-with-expose-standard', + }); }); it('does not generate config for plugins not exposing to client', async () => { - jest.doMock( - join('plugin-without-expose', 'server'), - () => ({ - config: { - schema: schema.object({ - serverProp: schema.string({ defaultValue: 'serverProp default value' }), - }), - }, - }), - { - virtual: true, - } - ); - const plugin = createPlugin('plugin-without-expose', { - path: 'plugin-without-expose', - configPath: 'path', + const prebootPlugin = createPlugin('plugin-without-expose-preboot', { + type: PluginType.preboot, + path: 'plugin-without-expose-preboot', + configPath: 'path-preboot', + }); + const standardPlugin = createPlugin('plugin-without-expose-standard', { + path: 'plugin-without-expose-standard', + configPath: 'path-standard', }); + for (const plugin of [prebootPlugin, standardPlugin]) { + jest.doMock( + join(plugin.path, 'server'), + () => ({ + config: { + schema: schema.object({ + serverProp: schema.string({ defaultValue: 'serverProp default value' }), + }), + }, + }), + { virtual: true } + ); + } + mockDiscover.mockReturnValue({ error$: from([]), - plugin$: from([plugin]), + plugin$: from([prebootPlugin, standardPlugin]), }); - mockPluginSystem.uiPlugins.mockReturnValue(new Map([pluginToDiscoveredEntry(plugin)])); + prebootMockPluginSystem.uiPlugins.mockReturnValue( + new Map([pluginToDiscoveredEntry(prebootPlugin)]) + ); + standardMockPluginSystem.uiPlugins.mockReturnValue( + new Map([pluginToDiscoveredEntry(standardPlugin)]) + ); - const { uiPlugins } = await pluginsService.discover({ environment: environmentSetup }); - expect([...uiPlugins.browserConfigs.entries()]).toHaveLength(0); + const { preboot, standard } = await pluginsService.discover({ + environment: environmentPreboot, + }); + expect(preboot.uiPlugins.browserConfigs.size).toBe(0); + expect(standard.uiPlugins.browserConfigs.size).toBe(0); }); }); - describe('#setup()', () => { + describe('plugin initialization', () => { beforeEach(() => { mockDiscover.mockReturnValue({ error$: from([]), plugin$: from([ - createPlugin('plugin-1', { - path: 'path-1', + createPlugin('plugin-1-preboot', { + type: PluginType.preboot, + path: 'path-1-preboot', version: 'version-1', - configPath: 'plugin1', + configPath: 'plugin1_preboot', }), - createPlugin('plugin-2', { - path: 'path-2', + createPlugin('plugin-1-standard', { + path: 'path-1-standard', + version: 'version-1', + configPath: 'plugin1_standard', + }), + createPlugin('plugin-2-preboot', { + type: PluginType.preboot, + path: 'path-2-preboot', version: 'version-2', - configPath: 'plugin2', + configPath: 'plugin2_preboot', + }), + createPlugin('plugin-2-standard', { + path: 'path-2-standard', + version: 'version-2', + configPath: 'plugin2_standard', }), ]), }); - mockPluginSystem.uiPlugins.mockReturnValue(new Map()); + prebootMockPluginSystem.uiPlugins.mockReturnValue(new Map()); + standardMockPluginSystem.uiPlugins.mockReturnValue(new Map()); }); - describe('uiPlugins.internal', () => { - it('contains internal properties for plugins', async () => { - config$.next({ plugins: { initialize: true }, plugin1: { enabled: false } }); - const { uiPlugins } = await pluginsService.discover({ environment: environmentSetup }); - expect(uiPlugins.internal).toMatchInlineSnapshot(` - Map { - "plugin-1" => Object { - "publicAssetsDir": /path-1/public/assets, - "publicTargetDir": /path-1/target/public, - "requiredBundles": Array [], - "version": "version-1", - }, - "plugin-2" => Object { - "publicAssetsDir": /path-2/public/assets, - "publicTargetDir": /path-2/target/public, - "requiredBundles": Array [], - "version": "version-2", - }, - } - `); + it('`uiPlugins.internal` contains internal properties for plugins', async () => { + config$.next({ + plugins: { initialize: true }, + plugin1_preboot: { enabled: false }, + plugin1_standard: { enabled: false }, }); - - it('includes disabled plugins', async () => { - config$.next({ plugins: { initialize: true }, plugin1: { enabled: false } }); - const { uiPlugins } = await pluginsService.discover({ environment: environmentSetup }); - expect([...uiPlugins.internal.keys()].sort()).toEqual(['plugin-1', 'plugin-2']); + const { preboot, standard } = await pluginsService.discover({ + environment: environmentPreboot, }); + expect(preboot.uiPlugins.internal).toMatchInlineSnapshot(` + Map { + "plugin-1-preboot" => Object { + "publicAssetsDir": /path-1-preboot/public/assets, + "publicTargetDir": /path-1-preboot/target/public, + "requiredBundles": Array [], + "version": "version-1", + }, + "plugin-2-preboot" => Object { + "publicAssetsDir": /path-2-preboot/public/assets, + "publicTargetDir": /path-2-preboot/target/public, + "requiredBundles": Array [], + "version": "version-2", + }, + } + `); + expect(standard.uiPlugins.internal).toMatchInlineSnapshot(` + Map { + "plugin-1-standard" => Object { + "publicAssetsDir": /path-1-standard/public/assets, + "publicTargetDir": /path-1-standard/target/public, + "requiredBundles": Array [], + "version": "version-1", + }, + "plugin-2-standard" => Object { + "publicAssetsDir": /path-2-standard/public/assets, + "publicTargetDir": /path-2-standard/target/public, + "requiredBundles": Array [], + "version": "version-2", + }, + } + `); }); - describe('plugin initialization', () => { - it('does initialize if plugins.initialize is true', async () => { - config$.next({ plugins: { initialize: true } }); - await pluginsService.discover({ environment: environmentSetup }); - const { initialized } = await pluginsService.setup(setupDeps); - expect(mockPluginSystem.setupPlugins).toHaveBeenCalled(); - expect(initialized).toBe(true); + it('`uiPlugins.internal` includes disabled plugins', async () => { + config$.next({ + plugins: { initialize: true }, + plugin1_preboot: { enabled: false }, + plugin1_standard: { enabled: false }, }); - - it('does not initialize if plugins.initialize is false', async () => { - config$.next({ plugins: { initialize: false } }); - await pluginsService.discover({ environment: environmentSetup }); - const { initialized } = await pluginsService.setup(setupDeps); - expect(mockPluginSystem.setupPlugins).not.toHaveBeenCalled(); - expect(initialized).toBe(false); + const { preboot, standard } = await pluginsService.discover({ + environment: environmentPreboot, }); + expect([...preboot.uiPlugins.internal.keys()].sort()).toMatchInlineSnapshot(` + Array [ + "plugin-1-preboot", + "plugin-2-preboot", + ] + `); + expect([...standard.uiPlugins.internal.keys()].sort()).toMatchInlineSnapshot(` + Array [ + "plugin-1-standard", + "plugin-2-standard", + ] + `); + }); + + it('#preboot does initialize `preboot` plugins if plugins.initialize is true', async () => { + config$.next({ plugins: { initialize: true } }); + await pluginsService.discover({ environment: environmentPreboot }); + await pluginsService.preboot(prebootDeps); + + expect(prebootMockPluginSystem.setupPlugins).toHaveBeenCalledTimes(1); + expect(prebootMockPluginSystem.setupPlugins).toHaveBeenCalledWith(prebootDeps); + expect(standardMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + }); + + it('#preboot does not initialize `preboot` plugins if plugins.initialize is false', async () => { + config$.next({ plugins: { initialize: false } }); + await pluginsService.discover({ environment: environmentPreboot }); + await pluginsService.preboot(prebootDeps); + + expect(prebootMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + }); + + it('#setup does initialize `standard` plugins if plugins.initialize is true', async () => { + config$.next({ plugins: { initialize: true } }); + await pluginsService.discover({ environment: environmentPreboot }); + await pluginsService.preboot(prebootDeps); + + const { initialized } = await pluginsService.setup(setupDeps); + expect(standardMockPluginSystem.setupPlugins).toHaveBeenCalledTimes(1); + expect(standardMockPluginSystem.setupPlugins).toHaveBeenCalledWith(setupDeps); + expect(initialized).toBe(true); + }); + + it('#setup does not initialize `standard` plugins if plugins.initialize is false', async () => { + config$.next({ plugins: { initialize: false } }); + await pluginsService.discover({ environment: environmentPreboot }); + await pluginsService.preboot(prebootDeps); + const { initialized } = await pluginsService.setup(setupDeps); + expect(standardMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + expect(prebootMockPluginSystem.setupPlugins).not.toHaveBeenCalled(); + expect(initialized).toBe(false); }); }); @@ -719,10 +1137,74 @@ describe('PluginsService', () => { }); }); + describe('#start()', () => { + beforeEach(() => { + mockDiscover.mockReturnValue({ + error$: from([]), + plugin$: from([ + createPlugin('plugin-1-preboot', { type: PluginType.preboot, path: 'path-1-preboot' }), + createPlugin('plugin-1-standard', { path: 'path-1-standard' }), + ]), + }); + }); + + it('does not try to stop `preboot` plugins and start `standard` ones if plugins.initialize is `false`', async () => { + config$.next({ plugins: { initialize: false } }); + + await pluginsService.discover({ environment: environmentPreboot }); + await pluginsService.preboot(prebootDeps); + await pluginsService.setup(setupDeps); + + const { contracts } = await pluginsService.start(startDeps); + expect(contracts).toBeInstanceOf(Map); + expect(contracts.size).toBe(0); + + expect(prebootMockPluginSystem.stopPlugins).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.startPlugins).not.toHaveBeenCalled(); + }); + + it('stops `preboot` plugins and starts `standard` ones', async () => { + await pluginsService.discover({ environment: environmentPreboot }); + await pluginsService.preboot(prebootDeps); + await pluginsService.setup(setupDeps); + + expect(prebootMockPluginSystem.stopPlugins).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.startPlugins).not.toHaveBeenCalled(); + + await pluginsService.start(startDeps); + + expect(prebootMockPluginSystem.stopPlugins).toHaveBeenCalledTimes(1); + expect(standardMockPluginSystem.stopPlugins).not.toHaveBeenCalled(); + + expect(standardMockPluginSystem.startPlugins).toHaveBeenCalledTimes(1); + expect(standardMockPluginSystem.startPlugins).toHaveBeenCalledWith(startDeps); + expect(prebootMockPluginSystem.startPlugins).not.toHaveBeenCalled(); + }); + }); + describe('#stop()', () => { it('`stop` stops plugins system', async () => { await pluginsService.stop(); - expect(mockPluginSystem.stopPlugins).toHaveBeenCalledTimes(1); + expect(standardMockPluginSystem.stopPlugins).toHaveBeenCalledTimes(1); + expect(prebootMockPluginSystem.stopPlugins).toHaveBeenCalledTimes(1); + }); + + it('`stop` does not try to stop preboot plugins system if it was stopped during `start`.', async () => { + await pluginsService.preboot(prebootDeps); + await pluginsService.setup(setupDeps); + + expect(prebootMockPluginSystem.stopPlugins).not.toHaveBeenCalled(); + expect(standardMockPluginSystem.stopPlugins).not.toHaveBeenCalled(); + + await pluginsService.start(startDeps); + + expect(prebootMockPluginSystem.stopPlugins).toHaveBeenCalledTimes(1); + expect(standardMockPluginSystem.stopPlugins).not.toHaveBeenCalled(); + + await pluginsService.stop(); + + expect(prebootMockPluginSystem.stopPlugins).toHaveBeenCalledTimes(1); + expect(standardMockPluginSystem.stopPlugins).toHaveBeenCalledTimes(1); }); }); }); diff --git a/src/core/server/plugins/plugins_service.ts b/src/core/server/plugins/plugins_service.ts index 99a9aaaddcb0bd..05bb60fb22c6db 100644 --- a/src/core/server/plugins/plugins_service.ts +++ b/src/core/server/plugins/plugins_service.ts @@ -8,20 +8,36 @@ import Path from 'path'; import { Observable } from 'rxjs'; -import { filter, first, map, concatMap, tap, toArray } from 'rxjs/operators'; -import { pick, getFlattenedObject } from '@kbn/std'; +import { concatMap, filter, first, map, tap, toArray } from 'rxjs/operators'; +import { getFlattenedObject, pick } from '@kbn/std'; import { CoreService } from '../../types'; import { CoreContext } from '../core_context'; import { Logger } from '../logging'; import { discover, PluginDiscoveryError, PluginDiscoveryErrorType } from './discovery'; import { PluginWrapper } from './plugin'; -import { DiscoveredPlugin, PluginConfigDescriptor, PluginName, InternalPluginInfo } from './types'; +import { + DiscoveredPlugin, + InternalPluginInfo, + PluginConfigDescriptor, + PluginDependencies, + PluginName, + PluginType, +} from './types'; import { PluginsConfig, PluginsConfigType } from './plugins_config'; import { PluginsSystem } from './plugins_system'; -import { InternalCoreSetup, InternalCoreStart } from '../internal_types'; +import { InternalCorePreboot, InternalCoreSetup, InternalCoreStart } from '../internal_types'; import { IConfigService } from '../config'; -import { InternalEnvironmentServiceSetup } from '../environment'; +import { InternalEnvironmentServicePreboot } from '../environment'; + +/** @internal */ +export type DiscoveredPlugins = { + [key in PluginType]: { + pluginTree: PluginDependencies; + pluginPaths: string[]; + uiPlugins: UiPlugins; + }; +}; /** @internal */ export interface PluginsServiceSetup { @@ -56,6 +72,9 @@ export interface PluginsServiceStart { contracts: Map; } +/** @internal */ +export type PluginsServicePrebootSetupDeps = InternalCorePreboot; + /** @internal */ export type PluginsServiceSetupDeps = InternalCoreSetup; @@ -64,29 +83,31 @@ export type PluginsServiceStartDeps = InternalCoreStart; /** @internal */ export interface PluginsServiceDiscoverDeps { - environment: InternalEnvironmentServiceSetup; + environment: InternalEnvironmentServicePreboot; } /** @internal */ export class PluginsService implements CoreService { private readonly log: Logger; - private readonly pluginsSystem: PluginsSystem; + private readonly prebootPluginsSystem = new PluginsSystem(this.coreContext, PluginType.preboot); + private arePrebootPluginsStopped = false; + private readonly prebootUiPluginInternalInfo = new Map(); + private readonly standardPluginsSystem = new PluginsSystem(this.coreContext, PluginType.standard); + private readonly standardUiPluginInternalInfo = new Map(); private readonly configService: IConfigService; private readonly config$: Observable; private readonly pluginConfigDescriptors = new Map(); - private readonly uiPluginInternalInfo = new Map(); private readonly pluginConfigUsageDescriptors = new Map>(); constructor(private readonly coreContext: CoreContext) { this.log = coreContext.logger.get('plugins-service'); - this.pluginsSystem = new PluginsSystem(coreContext); this.configService = coreContext.configService; this.config$ = coreContext.configService .atPath('plugins') .pipe(map((rawConfig) => new PluginsConfig(rawConfig, coreContext.env))); } - public async discover({ environment }: PluginsServiceDiscoverDeps) { + public async discover({ environment }: PluginsServiceDiscoverDeps): Promise { const config = await this.config$.pipe(first()).toPromise(); const { error$, plugin$ } = discover(config, this.coreContext, { @@ -96,16 +117,26 @@ export class PluginsService implements CoreService plugin.path), - uiPlugins: { - internal: this.uiPluginInternalInfo, - public: uiPlugins, - browserConfigs: this.generateUiPluginsConfigs(uiPlugins), + preboot: { + pluginPaths: this.prebootPluginsSystem.getPlugins().map((plugin) => plugin.path), + pluginTree: this.prebootPluginsSystem.getPluginDependencies(), + uiPlugins: { + internal: this.prebootUiPluginInternalInfo, + public: prebootUiPlugins, + browserConfigs: this.generateUiPluginsConfigs(prebootUiPlugins), + }, + }, + standard: { + pluginPaths: this.standardPluginsSystem.getPlugins().map((plugin) => plugin.path), + pluginTree: this.standardPluginsSystem.getPluginDependencies(), + uiPlugins: { + internal: this.standardUiPluginInternalInfo, + public: standardUiPlugins, + browserConfigs: this.generateUiPluginsConfigs(standardUiPlugins), + }, }, }; } @@ -114,6 +145,20 @@ export class PluginsService implements CoreService(); if (config.initialize) { - contracts = await this.pluginsSystem.setupPlugins(deps); - this.registerPluginStaticDirs(deps); + contracts = await this.standardPluginsSystem.setupPlugins(deps); + this.registerPluginStaticDirs(deps, this.standardUiPluginInternalInfo); } else { - this.log.info('Plugin initialization disabled.'); + this.log.info( + 'Skipping `setup` for `standard` plugins since plugin initialization is disabled.' + ); } return { @@ -135,13 +182,31 @@ export class PluginsService implements CoreService, parents: PluginName[] = [] - ): { enabled: true } | { enabled: false; missingDependencies: string[] } { + ): { enabled: true } | { enabled: false; missingOrIncompatibleDependencies: string[] } { const pluginInfo = pluginEnableStatuses.get(pluginName); if (pluginInfo === undefined || !pluginInfo.isEnabled) { return { enabled: false, - missingDependencies: [], + missingOrIncompatibleDependencies: [], }; } - const missingDependencies = pluginInfo.plugin.requiredPlugins + const missingOrIncompatibleDependencies = pluginInfo.plugin.requiredPlugins .filter((dep) => !parents.includes(dep)) .filter( (dependencyName) => + pluginEnableStatuses.get(dependencyName)?.plugin.manifest.type !== + pluginInfo.plugin.manifest.type || !this.shouldEnablePlugin(dependencyName, pluginEnableStatuses, [...parents, pluginName]) .enabled ); - if (missingDependencies.length === 0) { + if (missingOrIncompatibleDependencies.length === 0) { return { enabled: true, }; @@ -308,12 +390,15 @@ export class PluginsService implements CoreService + ) { + for (const [pluginName, pluginInfo] of uiPluginInternalInfo) { deps.http.registerStaticDir( `/plugins/${pluginName}/assets/{path*}`, pluginInfo.publicAssetsDir diff --git a/src/core/server/plugins/plugins_system.test.mocks.ts b/src/core/server/plugins/plugins_system.test.mocks.ts index 26175ab2b68a9e..1e1264ba76a809 100644 --- a/src/core/server/plugins/plugins_system.test.mocks.ts +++ b/src/core/server/plugins/plugins_system.test.mocks.ts @@ -6,9 +6,11 @@ * Side Public License, v 1. */ +export const mockCreatePluginPrebootSetupContext = jest.fn(); export const mockCreatePluginSetupContext = jest.fn(); export const mockCreatePluginStartContext = jest.fn(); jest.mock('./plugin_context', () => ({ + createPluginPrebootSetupContext: mockCreatePluginPrebootSetupContext, createPluginSetupContext: mockCreatePluginSetupContext, createPluginStartContext: mockCreatePluginStartContext, })); diff --git a/src/core/server/plugins/plugins_system.test.ts b/src/core/server/plugins/plugins_system.test.ts index abcd00f4e2daf7..e61c9c2002a12b 100644 --- a/src/core/server/plugins/plugins_system.test.ts +++ b/src/core/server/plugins/plugins_system.test.ts @@ -7,6 +7,7 @@ */ import { + mockCreatePluginPrebootSetupContext, mockCreatePluginSetupContext, mockCreatePluginStartContext, } from './plugins_system.test.mocks'; @@ -20,7 +21,7 @@ import { CoreContext } from '../core_context'; import { loggingSystemMock } from '../logging/logging_system.mock'; import { PluginWrapper } from './plugin'; -import { PluginName } from './types'; +import { PluginName, PluginType } from './types'; import { PluginsSystem } from './plugins_system'; import { coreMock } from '../mocks'; import { Logger } from '../logging'; @@ -32,7 +33,14 @@ function createPlugin( optional = [], server = true, ui = true, - }: { required?: string[]; optional?: string[]; server?: boolean; ui?: boolean } = {} + type = PluginType.standard, + }: { + required?: string[]; + optional?: string[]; + server?: boolean; + ui?: boolean; + type?: PluginType; + } = {} ): PluginWrapper { return new PluginWrapper({ path: 'some-path', @@ -41,6 +49,7 @@ function createPlugin( version: 'some-version', configPath: 'path', kibanaVersion: '7.0.0', + type, requiredPlugins: required, optionalPlugins: optional, requiredBundles: [], @@ -52,10 +61,11 @@ function createPlugin( }); } +const prebootDeps = coreMock.createInternalPreboot(); const setupDeps = coreMock.createInternalSetup(); const startDeps = coreMock.createInternalStart(); -let pluginsSystem: PluginsSystem; +let pluginsSystem: PluginsSystem; let configService: ReturnType; let logger: ReturnType; let env: Env; @@ -70,7 +80,7 @@ beforeEach(() => { coreContext = { coreId: Symbol(), env, logger, configService: configService as any }; - pluginsSystem = new PluginsSystem(coreContext); + pluginsSystem = new PluginsSystem(coreContext, PluginType.standard); }); test('can be setup even without plugins', async () => { @@ -80,6 +90,26 @@ test('can be setup even without plugins', async () => { expect(pluginsSetup.size).toBe(0); }); +test('throws if adding plugin with incompatible type', () => { + const prebootPlugin = createPlugin('plugin-preboot', { type: PluginType.preboot }); + const standardPlugin = createPlugin('plugin-standard'); + + const prebootPluginSystem = new PluginsSystem(coreContext, PluginType.preboot); + const standardPluginSystem = new PluginsSystem(coreContext, PluginType.standard); + + prebootPluginSystem.addPlugin(prebootPlugin); + expect(() => prebootPluginSystem.addPlugin(standardPlugin)).toThrowErrorMatchingInlineSnapshot( + `"Cannot add plugin with type \\"standard\\" to plugin system with type \\"preboot\\"."` + ); + expect(prebootPluginSystem.getPlugins()).toEqual([prebootPlugin]); + + standardPluginSystem.addPlugin(standardPlugin); + expect(() => standardPluginSystem.addPlugin(prebootPlugin)).toThrowErrorMatchingInlineSnapshot( + `"Cannot add plugin with type \\"preboot\\" to plugin system with type \\"standard\\"."` + ); + expect(standardPluginSystem.getPlugins()).toEqual([standardPlugin]); +}); + test('getPlugins returns the list of plugins', () => { const pluginA = createPlugin('plugin-a'); const pluginB = createPlugin('plugin-b'); @@ -293,6 +323,83 @@ test('correctly orders plugins and returns exposed values for "setup" and "start } }); +test('correctly orders preboot plugins and returns exposed values for "setup"', async () => { + const prebootPluginSystem = new PluginsSystem(coreContext, PluginType.preboot); + const plugins = new Map([ + [ + createPlugin('order-4', { type: PluginType.preboot, required: ['order-2'] }), + { 'order-2': 'added-as-2' }, + ], + [createPlugin('order-0', { type: PluginType.preboot }), {}], + [ + createPlugin('order-2', { + type: PluginType.preboot, + required: ['order-1'], + optional: ['order-0'], + }), + { 'order-1': 'added-as-3', 'order-0': 'added-as-1' }, + ], + [ + createPlugin('order-1', { type: PluginType.preboot, required: ['order-0'] }), + { 'order-0': 'added-as-1' }, + ], + [ + createPlugin('order-3', { + type: PluginType.preboot, + required: ['order-2'], + optional: ['missing-dep'], + }), + { 'order-2': 'added-as-2' }, + ], + ] as Array<[PluginWrapper, Record]>); + + const setupContextMap = new Map(); + [...plugins.keys()].forEach((plugin, index) => { + jest.spyOn(plugin, 'setup').mockResolvedValue(`added-as-${index}`); + setupContextMap.set(plugin.name, `setup-for-${plugin.name}`); + prebootPluginSystem.addPlugin(plugin); + }); + + mockCreatePluginPrebootSetupContext.mockImplementation((context, deps, plugin) => + setupContextMap.get(plugin.name) + ); + + expect([...(await prebootPluginSystem.setupPlugins(prebootDeps))]).toMatchInlineSnapshot(` + Array [ + Array [ + "order-0", + "added-as-1", + ], + Array [ + "order-1", + "added-as-3", + ], + Array [ + "order-2", + "added-as-2", + ], + Array [ + "order-3", + "added-as-4", + ], + Array [ + "order-4", + "added-as-0", + ], + ] + `); + + for (const [plugin, deps] of plugins) { + expect(mockCreatePluginPrebootSetupContext).toHaveBeenCalledWith( + coreContext, + prebootDeps, + plugin + ); + expect(plugin.setup).toHaveBeenCalledTimes(1); + expect(plugin.setup).toHaveBeenCalledWith(setupContextMap.get(plugin.name), deps); + } +}); + test('`setupPlugins` only setups plugins that have server side', async () => { const firstPluginToRun = createPlugin('order-0'); const secondPluginNotToRun = createPlugin('order-not-run', { server: false }); @@ -399,6 +506,21 @@ test('can start without plugins', async () => { expect(pluginsStart.size).toBe(0); }); +test('cannot start preboot plugins', async () => { + const prebootPlugin = createPlugin('order-0', { type: PluginType.preboot }); + jest.spyOn(prebootPlugin, 'setup').mockResolvedValue({}); + jest.spyOn(prebootPlugin, 'start').mockResolvedValue({}); + + const prebootPluginSystem = new PluginsSystem(coreContext, PluginType.preboot); + prebootPluginSystem.addPlugin(prebootPlugin); + await prebootPluginSystem.setupPlugins(prebootDeps); + + await expect( + prebootPluginSystem.startPlugins(startDeps) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Preboot plugins cannot be started."`); + expect(prebootPlugin.start).not.toHaveBeenCalled(); +}); + test('`startPlugins` only starts plugins that were setup', async () => { const firstPluginToRun = createPlugin('order-0'); const secondPluginNotToRun = createPlugin('order-not-run', { server: false }); @@ -525,7 +647,7 @@ describe('asynchronous plugins', () => { }) ); coreContext = { coreId: Symbol(), env, logger, configService: configService as any }; - pluginsSystem = new PluginsSystem(coreContext); + pluginsSystem = new PluginsSystem(coreContext, PluginType.standard); const syncPlugin = createPlugin('sync-plugin'); jest.spyOn(syncPlugin, 'setup').mockReturnValue('setup-sync'); diff --git a/src/core/server/plugins/plugins_system.ts b/src/core/server/plugins/plugins_system.ts index f6327d4eabf43c..4a156c5fbb976d 100644 --- a/src/core/server/plugins/plugins_system.ts +++ b/src/core/server/plugins/plugins_system.ts @@ -10,25 +10,38 @@ import { withTimeout, isPromise } from '@kbn/std'; import { CoreContext } from '../core_context'; import { Logger } from '../logging'; import { PluginWrapper } from './plugin'; -import { DiscoveredPlugin, PluginName } from './types'; -import { createPluginSetupContext, createPluginStartContext } from './plugin_context'; -import { PluginsServiceSetupDeps, PluginsServiceStartDeps } from './plugins_service'; -import { PluginDependencies } from '.'; +import { DiscoveredPlugin, PluginDependencies, PluginName, PluginType } from './types'; +import { + createPluginPrebootSetupContext, + createPluginSetupContext, + createPluginStartContext, +} from './plugin_context'; +import { + PluginsServicePrebootSetupDeps, + PluginsServiceSetupDeps, + PluginsServiceStartDeps, +} from './plugins_service'; const Sec = 1000; /** @internal */ -export class PluginsSystem { +export class PluginsSystem { private readonly plugins = new Map(); private readonly log: Logger; // `satup`, the past-tense version of the noun `setup`. private readonly satupPlugins: PluginName[] = []; - constructor(private readonly coreContext: CoreContext) { - this.log = coreContext.logger.get('plugins-system'); + constructor(private readonly coreContext: CoreContext, public readonly type: T) { + this.log = coreContext.logger.get('plugins-system', this.type); } public addPlugin(plugin: PluginWrapper) { + if (plugin.manifest.type !== this.type) { + throw new Error( + `Cannot add plugin with type "${plugin.manifest.type}" to plugin system with type "${this.type}".` + ); + } + this.plugins.set(plugin.name, plugin); } @@ -67,7 +80,9 @@ export class PluginsSystem { return { asNames, asOpaqueIds }; } - public async setupPlugins(deps: PluginsServiceSetupDeps) { + public async setupPlugins( + deps: T extends PluginType.preboot ? PluginsServicePrebootSetupDeps : PluginsServiceSetupDeps + ): Promise> { const contracts = new Map(); if (this.plugins.size === 0) { return contracts; @@ -95,11 +110,23 @@ export class PluginsSystem { return depContracts; }, {} as Record); + let pluginSetupContext; + if (this.type === PluginType.preboot) { + pluginSetupContext = createPluginPrebootSetupContext( + this.coreContext, + deps as PluginsServicePrebootSetupDeps, + plugin + ); + } else { + pluginSetupContext = createPluginSetupContext( + this.coreContext, + deps as PluginsServiceSetupDeps, + plugin + ); + } + let contract: unknown; - const contractOrPromise = plugin.setup( - createPluginSetupContext(this.coreContext, deps, plugin), - pluginDepContracts - ); + const contractOrPromise = plugin.setup(pluginSetupContext, pluginDepContracts); if (isPromise(contractOrPromise)) { if (this.coreContext.env.mode.dev) { this.log.warn( @@ -130,6 +157,10 @@ export class PluginsSystem { } public async startPlugins(deps: PluginsServiceStartDeps) { + if (this.type === PluginType.preboot) { + throw new Error('Preboot plugins cannot be started.'); + } + const contracts = new Map(); if (this.satupPlugins.length === 0) { return contracts; @@ -222,6 +253,7 @@ export class PluginsSystem { pluginName, { id: pluginName, + type: plugin.manifest.type, configPath: plugin.manifest.configPath, requiredPlugins: plugin.manifest.requiredPlugins.filter((p) => uiPluginNames.includes(p) diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index 0cdc806e997ef7..b0edcbdfd8677b 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -16,7 +16,7 @@ import { LoggerFactory } from '../logging'; import { KibanaConfigType } from '../kibana_config'; import { ElasticsearchConfigType } from '../elasticsearch/elasticsearch_config'; import { SavedObjectsConfigType } from '../saved_objects/saved_objects_config'; -import { CoreSetup, CoreStart } from '..'; +import { CorePreboot, CoreSetup, CoreStart } from '..'; type Maybe = T | undefined; @@ -116,6 +116,18 @@ export type PluginName = string; /** @public */ export type PluginOpaqueId = symbol; +/** @public */ +export enum PluginType { + /** + * Preboot plugins are special-purpose plugins that only function during preboot stage. + */ + preboot = 'preboot', + /** + * Standard plugins are plugins that start to function as soon as Kibana is fully booted and are active until it shuts down. + */ + standard = 'standard', +} + /** @internal */ export interface PluginDependencies { asNames: ReadonlyMap; @@ -149,6 +161,11 @@ export interface PluginManifest { */ readonly kibanaVersion: string; + /** + * Type of the plugin, defaults to `standard`. + */ + readonly type: PluginType; + /** * Root {@link ConfigPath | configuration path} used by the plugin, defaults * to "id" in snake_case format. @@ -247,6 +264,11 @@ export interface DiscoveredPlugin { */ readonly configPath: ConfigPath; + /** + * Type of the plugin, defaults to `standard`. + */ + readonly type: PluginType; + /** * An optional list of the other plugins that **must be** installed and enabled * for this plugin to function properly. @@ -296,7 +318,18 @@ export interface InternalPluginInfo { } /** - * The interface that should be returned by a `PluginInitializer`. + * The interface that should be returned by a `PluginInitializer` for a `preboot` plugin. + * + * @public + */ +export interface PrebootPlugin { + setup(core: CorePreboot, plugins: TPluginsSetup): TSetup; + + stop?(): void; +} + +/** + * The interface that should be returned by a `PluginInitializer` for a `standard` plugin. * * @public */ @@ -361,6 +394,7 @@ export interface PluginInitializerContext { mode: EnvironmentMode; packageInfo: Readonly; instanceUuid: string; + configs: readonly string[]; }; /** * {@link LoggerFactory | logger factory} instance already bound to the plugin's logging context @@ -471,4 +505,5 @@ export type PluginInitializer< core: PluginInitializerContext ) => | Plugin + | PrebootPlugin | AsyncPlugin; diff --git a/src/core/server/preboot/index.ts b/src/core/server/preboot/index.ts new file mode 100644 index 00000000000000..2b7f25538dcb1c --- /dev/null +++ b/src/core/server/preboot/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { PrebootService } from './preboot_service'; +export type { InternalPrebootServicePreboot, PrebootServicePreboot } from './types'; diff --git a/src/core/server/preboot/preboot_service.mock.ts b/src/core/server/preboot/preboot_service.mock.ts new file mode 100644 index 00000000000000..acdd9458a462d1 --- /dev/null +++ b/src/core/server/preboot/preboot_service.mock.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { PublicMethodsOf } from '@kbn/utility-types'; +import type { InternalPrebootServicePreboot, PrebootServicePreboot } from './types'; +import { PrebootService } from './preboot_service'; + +export type InternalPrebootServicePrebootMock = jest.Mocked; +export type PrebootServicePrebootMock = jest.Mocked; + +const createInternalPrebootContractMock = () => { + const mock: InternalPrebootServicePrebootMock = { + isSetupOnHold: jest.fn(), + holdSetupUntilResolved: jest.fn(), + waitUntilCanSetup: jest.fn(), + }; + return mock; +}; + +const createPrebootContractMock = () => { + const mock: PrebootServicePrebootMock = { + isSetupOnHold: jest.fn(), + holdSetupUntilResolved: jest.fn(), + }; + + return mock; +}; + +type PrebootServiceContract = PublicMethodsOf; + +const createPrebootServiceMock = () => { + const mocked: jest.Mocked = { + preboot: jest.fn(), + stop: jest.fn(), + }; + mocked.preboot.mockReturnValue(createInternalPrebootContractMock()); + return mocked; +}; + +export const prebootServiceMock = { + create: createPrebootServiceMock, + createInternalPrebootContract: createInternalPrebootContractMock, + createPrebootContract: createPrebootContractMock, +}; diff --git a/src/core/server/preboot/preboot_service.test.ts b/src/core/server/preboot/preboot_service.test.ts new file mode 100644 index 00000000000000..dd4b1cb7d1df0b --- /dev/null +++ b/src/core/server/preboot/preboot_service.test.ts @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { nextTick } from '@kbn/test/jest'; +import { REPO_ROOT } from '@kbn/dev-utils'; +import { LoggerFactory } from '@kbn/logging'; +import { Env } from '@kbn/config'; +import { getEnvOptions } from '../config/mocks'; +import { configServiceMock, loggingSystemMock } from '../mocks'; + +import { PrebootService } from './preboot_service'; + +describe('PrebootService', () => { + describe('#preboot()', () => { + let service: PrebootService; + let logger: LoggerFactory; + beforeEach(() => { + logger = loggingSystemMock.create(); + service = new PrebootService({ + configService: configServiceMock.create(), + env: Env.createDefault(REPO_ROOT, getEnvOptions()), + logger, + coreId: Symbol(), + }); + }); + + it('returns a proper contract', () => { + expect(service.preboot()).toMatchInlineSnapshot(` + Object { + "holdSetupUntilResolved": [Function], + "isSetupOnHold": [Function], + "waitUntilCanSetup": [Function], + } + `); + }); + + it('#isSetupOnHold correctly determines if `setup` is on hold', async () => { + const preboot = service.preboot(); + + expect(preboot.isSetupOnHold()).toBe(false); + + let resolveFirstPromise: (value?: { shouldReloadConfig: boolean }) => void; + preboot.holdSetupUntilResolved( + 'some-plugin', + 'some-reason', + new Promise<{ shouldReloadConfig: boolean } | undefined>((resolve) => { + resolveFirstPromise = resolve; + }) + ); + + let resolveSecondPromise: (value?: { shouldReloadConfig: boolean }) => void; + preboot.holdSetupUntilResolved( + 'some-other-plugin', + 'some-other-reason', + new Promise<{ shouldReloadConfig: boolean } | undefined>((resolve) => { + resolveSecondPromise = resolve; + }) + ); + + expect(preboot.isSetupOnHold()).toBe(true); + const waitUntilPromise = preboot.waitUntilCanSetup(); + + resolveFirstPromise!(); + await nextTick(); + expect(preboot.isSetupOnHold()).toBe(true); + + resolveSecondPromise!(); + await nextTick(); + expect(preboot.isSetupOnHold()).toBe(false); + + await expect(waitUntilPromise).resolves.toEqual({ shouldReloadConfig: false }); + }); + + it('#holdSetupUntilResolved logs a reason', async () => { + const preboot = service.preboot(); + + preboot.holdSetupUntilResolved( + 'some-plugin', + 'some-reason', + Promise.resolve({ shouldReloadConfig: true }) + ); + preboot.holdSetupUntilResolved( + 'some-other-plugin', + 'some-other-reason', + Promise.resolve(undefined) + ); + + expect(loggingSystemMock.collect(logger).info).toMatchInlineSnapshot(` + Array [ + Array [ + "\\"some-plugin\\" plugin is holding setup: some-reason", + ], + Array [ + "\\"some-other-plugin\\" plugin is holding setup: some-other-reason", + ], + ] + `); + + await expect(preboot.waitUntilCanSetup()).resolves.toEqual({ shouldReloadConfig: true }); + }); + + it('#holdSetupUntilResolved does not allow to register new promises after #waitUntilCanSetup is called', async () => { + const preboot = service.preboot(); + + preboot.holdSetupUntilResolved( + 'some-plugin', + 'some-reason', + Promise.resolve({ shouldReloadConfig: true }) + ); + + const waitUntilPromise = preboot.waitUntilCanSetup(); + + expect(() => + preboot.holdSetupUntilResolved( + 'some-other-plugin', + 'some-other-reason', + Promise.resolve(undefined) + ) + ).toThrowErrorMatchingInlineSnapshot(`"Cannot hold boot at this stage."`); + + expect(loggingSystemMock.collect(logger).info).toMatchInlineSnapshot(` + Array [ + Array [ + "\\"some-plugin\\" plugin is holding setup: some-reason", + ], + ] + `); + + await expect(waitUntilPromise).resolves.toEqual({ shouldReloadConfig: true }); + }); + + it('#waitUntilCanSetup returns `shouldReloadConfig` set to `true` if at least one promise did it', async () => { + const preboot = service.preboot(); + + expect(preboot.isSetupOnHold()).toBe(false); + + let resolveFirstPromise: (value?: { shouldReloadConfig: boolean }) => void; + preboot.holdSetupUntilResolved( + 'some-plugin', + 'some-reason', + new Promise<{ shouldReloadConfig: boolean } | undefined>((resolve) => { + resolveFirstPromise = resolve; + }) + ); + + let resolveSecondPromise: (value?: { shouldReloadConfig: boolean }) => void; + preboot.holdSetupUntilResolved( + 'some-other-plugin', + 'some-other-reason', + new Promise<{ shouldReloadConfig: boolean } | undefined>((resolve) => { + resolveSecondPromise = resolve; + }) + ); + + expect(preboot.isSetupOnHold()).toBe(true); + const waitUntilPromise = preboot.waitUntilCanSetup(); + + resolveFirstPromise!({ shouldReloadConfig: true }); + await nextTick(); + expect(preboot.isSetupOnHold()).toBe(true); + + resolveSecondPromise!({ shouldReloadConfig: false }); + await nextTick(); + expect(preboot.isSetupOnHold()).toBe(false); + + await expect(waitUntilPromise).resolves.toEqual({ shouldReloadConfig: true }); + }); + + it('#waitUntilCanSetup is rejected if at least one promise is rejected', async () => { + const preboot = service.preboot(); + + preboot.holdSetupUntilResolved( + 'some-plugin', + 'some-reason', + Promise.resolve({ shouldReloadConfig: true }) + ); + preboot.holdSetupUntilResolved( + 'some-other-plugin', + 'some-other-reason', + Promise.reject('Uh oh!') + ); + + await expect(preboot.waitUntilCanSetup()).rejects.toBe('Uh oh!'); + }); + }); +}); diff --git a/src/core/server/preboot/preboot_service.ts b/src/core/server/preboot/preboot_service.ts new file mode 100644 index 00000000000000..4313541ef91d3a --- /dev/null +++ b/src/core/server/preboot/preboot_service.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { CoreContext } from '../core_context'; +import { InternalPrebootServicePreboot } from './types'; + +/** @internal */ +export class PrebootService { + private readonly promiseList: Array> = []; + private waitUntilCanSetupPromise?: Promise<{ shouldReloadConfig: boolean }>; + private isSetupOnHold = false; + private readonly log = this.core.logger.get('preboot'); + + constructor(private readonly core: CoreContext) {} + + public preboot(): InternalPrebootServicePreboot { + return { + isSetupOnHold: () => this.isSetupOnHold, + holdSetupUntilResolved: (pluginName, reason, promise) => { + if (this.waitUntilCanSetupPromise) { + throw new Error('Cannot hold boot at this stage.'); + } + + this.log.info(`"${pluginName}" plugin is holding setup: ${reason}`); + + this.isSetupOnHold = true; + + this.promiseList.push(promise); + }, + waitUntilCanSetup: () => { + if (!this.waitUntilCanSetupPromise) { + this.waitUntilCanSetupPromise = Promise.all(this.promiseList) + .then((results) => ({ + shouldReloadConfig: results.some((result) => result?.shouldReloadConfig), + })) + .catch((err) => { + this.log.error(err); + throw err; + }) + .finally(() => (this.isSetupOnHold = false)); + } + + return this.waitUntilCanSetupPromise; + }, + }; + } + + public stop() { + this.isSetupOnHold = false; + this.promiseList.length = 0; + this.waitUntilCanSetupPromise = undefined; + } +} diff --git a/src/core/server/preboot/types.ts b/src/core/server/preboot/types.ts new file mode 100644 index 00000000000000..61abc327c9ddbe --- /dev/null +++ b/src/core/server/preboot/types.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PluginName } from '..'; + +/** @internal */ +export interface InternalPrebootServicePreboot { + /** + * Indicates whether Kibana is currently on hold and cannot proceed to `setup` yet. + */ + readonly isSetupOnHold: () => boolean; + + /** + * Registers a `Promise` as a precondition before Kibana can proceed to `setup`. This method can be invoked multiple + * times and from multiple `preboot` plugins. Kibana will proceed to `setup` only when all registered `Promise` are + * resolved, or it will shut down if any of them are rejected. + * @param pluginName Name of the plugin that needs to hold `setup`. + * @param reason A string that explains the reason why this promise should hold `setup`. It's supposed to be a human + * readable string that will be recorded in the logs or standard output. + * @param promise A `Promise` that should resolved before Kibana can proceed to `setup`. + */ + readonly holdSetupUntilResolved: ( + pluginName: PluginName, + reason: string, + promise: Promise<{ shouldReloadConfig: boolean } | undefined> + ) => void; + + /** + * Returns a `Promise` that is resolved only when all `Promise` instances registered with {@link holdSetupUntilResolved} + * are resolved, or rejected if any of them are rejected. If the supplied `Promise` resolves to an object with the + * `shouldReloadConfig` property set to `true`, it indicates that Kibana configuration might have changed and Kibana + * needs to reload it from the disk. + */ + readonly waitUntilCanSetup: () => Promise<{ shouldReloadConfig: boolean }>; +} + +/** + * Kibana Preboot Service allows to control the boot flow of Kibana. Preboot plugins can use it to hold the boot until certain condition is met. + * + * @example + * A plugin can supply a `Promise` to a `holdSetupUntilResolved` method to signal Kibana to initialize and start `standard` plugins only after this + * `Promise` is resolved. If `Promise` is rejected, Kibana will shut down. + * ```ts + * core.preboot.holdSetupUntilResolved('Just waiting for 5 seconds', + * new Promise((resolve) => { + * setTimeout(resolve, 5000); + * }) + * ); + * ``` + * + * If the supplied `Promise` resolves to an object with the `shouldReloadConfig` property set to `true`, Kibana will also reload its configuration from disk. + * ```ts + * let completeSetup: (result: { shouldReloadConfig: boolean }) => void; + * core.preboot.holdSetupUntilResolved('Just waiting for 5 seconds before reloading configuration', + * new Promise<{ shouldReloadConfig: boolean }>((resolve) => { + * setTimeout(() => resolve({ shouldReloadConfig: true }), 5000); + * }) + * ); + * ``` + * @public + */ +export interface PrebootServicePreboot { + /** + * Indicates whether Kibana is currently on hold and cannot proceed to `setup` yet. + */ + readonly isSetupOnHold: () => boolean; + + /** + * Registers a `Promise` as a precondition before Kibana can proceed to `setup`. This method can be invoked multiple + * times and from multiple `preboot` plugins. Kibana will proceed to `setup` only when all registered `Promises` + * instances are resolved, or it will shut down if any of them is rejected. + * @param reason A string that explains the reason why this promise should hold `setup`. It's supposed to be a human + * readable string that will be recorded in the logs or standard output. + * @param promise A `Promise` that should resolved before Kibana can proceed to `setup`. + */ + readonly holdSetupUntilResolved: ( + reason: string, + promise: Promise<{ shouldReloadConfig: boolean } | undefined> + ) => void; +} diff --git a/src/core/server/preboot_core_route_handler_context.test.ts b/src/core/server/preboot_core_route_handler_context.test.ts new file mode 100644 index 00000000000000..8d090d86446378 --- /dev/null +++ b/src/core/server/preboot_core_route_handler_context.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PrebootCoreRouteHandlerContext } from './preboot_core_route_handler_context'; +import { coreMock } from './mocks'; + +describe('#uiSettings', () => { + describe('#client', () => { + test('returns the results of corePreboot.uiSettings.createDefaultsClient', () => { + const corePreboot = coreMock.createInternalPreboot(); + const context = new PrebootCoreRouteHandlerContext(corePreboot); + + const client = context.uiSettings.client; + const [{ value: mockResult }] = corePreboot.uiSettings.createDefaultsClient.mock.results; + expect(client).toBe(mockResult); + }); + + test('only creates one instance', () => { + const corePreboot = coreMock.createInternalPreboot(); + const context = new PrebootCoreRouteHandlerContext(corePreboot); + + const client1 = context.uiSettings.client; + const client2 = context.uiSettings.client; + + expect(corePreboot.uiSettings.createDefaultsClient).toHaveBeenCalledTimes(1); + const [{ value: mockResult }] = corePreboot.uiSettings.createDefaultsClient.mock.results; + expect(client1).toBe(mockResult); + expect(client2).toBe(mockResult); + }); + }); +}); diff --git a/src/core/server/preboot_core_route_handler_context.ts b/src/core/server/preboot_core_route_handler_context.ts new file mode 100644 index 00000000000000..63378046e80500 --- /dev/null +++ b/src/core/server/preboot_core_route_handler_context.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +// eslint-disable-next-line max-classes-per-file +import { InternalCorePreboot } from './internal_types'; +import { IUiSettingsClient } from './ui_settings'; + +class PrebootCoreUiSettingsRouteHandlerContext { + constructor(public readonly client: IUiSettingsClient) {} +} + +export class PrebootCoreRouteHandlerContext { + readonly uiSettings: PrebootCoreUiSettingsRouteHandlerContext; + + constructor(private readonly corePreboot: InternalCorePreboot) { + this.uiSettings = new PrebootCoreUiSettingsRouteHandlerContext( + this.corePreboot.uiSettings.createDefaultsClient() + ); + } +} diff --git a/src/core/server/rendering/__mocks__/params.ts b/src/core/server/rendering/__mocks__/params.ts index d790e4a6799881..091d185cceefce 100644 --- a/src/core/server/rendering/__mocks__/params.ts +++ b/src/core/server/rendering/__mocks__/params.ts @@ -12,13 +12,17 @@ import { pluginServiceMock } from '../../plugins/plugins_service.mock'; import { statusServiceMock } from '../../status/status_service.mock'; const context = mockCoreContext.create(); -const http = httpServiceMock.createInternalSetupContract(); -const uiPlugins = pluginServiceMock.createUiPlugins(); +const httpPreboot = httpServiceMock.createInternalPrebootContract(); +const httpSetup = httpServiceMock.createInternalSetupContract(); const status = statusServiceMock.createInternalSetupContract(); export const mockRenderingServiceParams = context; +export const mockRenderingPrebootDeps = { + http: httpPreboot, + uiPlugins: pluginServiceMock.createUiPlugins(), +}; export const mockRenderingSetupDeps = { - http, - uiPlugins, + http: httpSetup, + uiPlugins: pluginServiceMock.createUiPlugins(), status, }; diff --git a/src/core/server/rendering/__mocks__/rendering_service.ts b/src/core/server/rendering/__mocks__/rendering_service.ts index 56131d77b23ebe..76558a999a3746 100644 --- a/src/core/server/rendering/__mocks__/rendering_service.ts +++ b/src/core/server/rendering/__mocks__/rendering_service.ts @@ -8,17 +8,22 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import type { RenderingService as Service } from '../rendering_service'; -import type { InternalRenderingServiceSetup } from '../types'; +import type { InternalRenderingServicePreboot, InternalRenderingServiceSetup } from '../types'; import { mockRenderingServiceParams } from './params'; type IRenderingService = PublicMethodsOf; +export const prebootMock: jest.Mocked = { + render: jest.fn(), +}; export const setupMock: jest.Mocked = { render: jest.fn(), }; +export const mockPreboot = jest.fn().mockResolvedValue(prebootMock); export const mockSetup = jest.fn().mockResolvedValue(setupMock); export const mockStop = jest.fn(); export const mockRenderingService: jest.Mocked = { + preboot: mockPreboot, setup: mockSetup, stop: mockStop, }; diff --git a/src/core/server/rendering/__snapshots__/rendering_service.test.ts.snap b/src/core/server/rendering/__snapshots__/rendering_service.test.ts.snap index f6b39ea24262b4..495a38a1af5bf3 100644 --- a/src/core/server/rendering/__snapshots__/rendering_service.test.ts.snap +++ b/src/core/server/rendering/__snapshots__/rendering_service.test.ts.snap @@ -1,5 +1,264 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`RenderingService preboot() render() renders "core" from legacy request 1`] = ` +Object { + "anonymousStatusPage": false, + "basePath": "/mock-server-basepath", + "branch": Any, + "buildNumber": Any, + "csp": Object { + "warnLegacyBrowsers": true, + }, + "env": Object { + "mode": Object { + "dev": Any, + "name": Any, + "prod": Any, + }, + "packageInfo": Object { + "branch": Any, + "buildNum": Any, + "buildSha": Any, + "dist": Any, + "version": Any, + }, + }, + "externalUrl": Object { + "policy": Array [ + Object { + "allow": true, + }, + ], + }, + "i18n": Object { + "translationsUrl": "/mock-server-basepath/translations/en.json", + }, + "legacyMetadata": Object { + "uiSettings": Object { + "defaults": Object { + "registered": Object { + "name": "title", + }, + }, + "user": Object {}, + }, + }, + "publicBaseUrl": "http://myhost.com/mock-server-basepath", + "serverBasePath": "/mock-server-basepath", + "uiPlugins": Array [], + "vars": Object {}, + "version": Any, +} +`; + +exports[`RenderingService preboot() render() renders "core" page 1`] = ` +Object { + "anonymousStatusPage": false, + "basePath": "/mock-server-basepath", + "branch": Any, + "buildNumber": Any, + "csp": Object { + "warnLegacyBrowsers": true, + }, + "env": Object { + "mode": Object { + "dev": Any, + "name": Any, + "prod": Any, + }, + "packageInfo": Object { + "branch": Any, + "buildNum": Any, + "buildSha": Any, + "dist": Any, + "version": Any, + }, + }, + "externalUrl": Object { + "policy": Array [ + Object { + "allow": true, + }, + ], + }, + "i18n": Object { + "translationsUrl": "/mock-server-basepath/translations/en.json", + }, + "legacyMetadata": Object { + "uiSettings": Object { + "defaults": Object { + "registered": Object { + "name": "title", + }, + }, + "user": Object {}, + }, + }, + "publicBaseUrl": "http://myhost.com/mock-server-basepath", + "serverBasePath": "/mock-server-basepath", + "uiPlugins": Array [], + "vars": Object {}, + "version": Any, +} +`; + +exports[`RenderingService preboot() render() renders "core" page driven by settings 1`] = ` +Object { + "anonymousStatusPage": false, + "basePath": "/mock-server-basepath", + "branch": Any, + "buildNumber": Any, + "csp": Object { + "warnLegacyBrowsers": true, + }, + "env": Object { + "mode": Object { + "dev": Any, + "name": Any, + "prod": Any, + }, + "packageInfo": Object { + "branch": Any, + "buildNum": Any, + "buildSha": Any, + "dist": Any, + "version": Any, + }, + }, + "externalUrl": Object { + "policy": Array [ + Object { + "allow": true, + }, + ], + }, + "i18n": Object { + "translationsUrl": "/mock-server-basepath/translations/en.json", + }, + "legacyMetadata": Object { + "uiSettings": Object { + "defaults": Object { + "registered": Object { + "name": "title", + }, + }, + "user": Object { + "theme:darkMode": Object { + "userValue": true, + }, + }, + }, + }, + "publicBaseUrl": "http://myhost.com/mock-server-basepath", + "serverBasePath": "/mock-server-basepath", + "uiPlugins": Array [], + "vars": Object {}, + "version": Any, +} +`; + +exports[`RenderingService preboot() render() renders "core" page for blank basepath 1`] = ` +Object { + "anonymousStatusPage": false, + "basePath": "", + "branch": Any, + "buildNumber": Any, + "csp": Object { + "warnLegacyBrowsers": true, + }, + "env": Object { + "mode": Object { + "dev": Any, + "name": Any, + "prod": Any, + }, + "packageInfo": Object { + "branch": Any, + "buildNum": Any, + "buildSha": Any, + "dist": Any, + "version": Any, + }, + }, + "externalUrl": Object { + "policy": Array [ + Object { + "allow": true, + }, + ], + }, + "i18n": Object { + "translationsUrl": "/translations/en.json", + }, + "legacyMetadata": Object { + "uiSettings": Object { + "defaults": Object { + "registered": Object { + "name": "title", + }, + }, + "user": Object {}, + }, + }, + "publicBaseUrl": "http://myhost.com/mock-server-basepath", + "serverBasePath": "/mock-server-basepath", + "uiPlugins": Array [], + "vars": Object {}, + "version": Any, +} +`; + +exports[`RenderingService preboot() render() renders "core" with excluded user settings 1`] = ` +Object { + "anonymousStatusPage": false, + "basePath": "/mock-server-basepath", + "branch": Any, + "buildNumber": Any, + "csp": Object { + "warnLegacyBrowsers": true, + }, + "env": Object { + "mode": Object { + "dev": Any, + "name": Any, + "prod": Any, + }, + "packageInfo": Object { + "branch": Any, + "buildNum": Any, + "buildSha": Any, + "dist": Any, + "version": Any, + }, + }, + "externalUrl": Object { + "policy": Array [ + Object { + "allow": true, + }, + ], + }, + "i18n": Object { + "translationsUrl": "/mock-server-basepath/translations/en.json", + }, + "legacyMetadata": Object { + "uiSettings": Object { + "defaults": Object { + "registered": Object { + "name": "title", + }, + }, + "user": Object {}, + }, + }, + "publicBaseUrl": "http://myhost.com/mock-server-basepath", + "serverBasePath": "/mock-server-basepath", + "uiPlugins": Array [], + "vars": Object {}, + "version": Any, +} +`; + exports[`RenderingService setup() render() renders "core" from legacy request 1`] = ` Object { "anonymousStatusPage": false, diff --git a/src/core/server/rendering/bootstrap/get_plugin_bundle_paths.test.ts b/src/core/server/rendering/bootstrap/get_plugin_bundle_paths.test.ts index 0abd8fd5a00576..67f8507b9b700f 100644 --- a/src/core/server/rendering/bootstrap/get_plugin_bundle_paths.test.ts +++ b/src/core/server/rendering/bootstrap/get_plugin_bundle_paths.test.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { InternalPluginInfo, UiPlugins } from '../../plugins'; +import { InternalPluginInfo, PluginType, UiPlugins } from '../../plugins'; import { getPluginsBundlePaths } from './get_plugin_bundle_paths'; const createUiPlugins = (pluginDeps: Record) => { @@ -26,6 +26,7 @@ const createUiPlugins = (pluginDeps: Record) => { uiPlugins.public.set(pluginId, { id: pluginId, configPath: 'config-path', + type: PluginType.standard, optionalPlugins: [], requiredPlugins: [], requiredBundles: deps, diff --git a/src/core/server/rendering/rendering_service.mock.ts b/src/core/server/rendering/rendering_service.mock.ts index 81418c58175f5b..3d8213da62c6d9 100644 --- a/src/core/server/rendering/rendering_service.mock.ts +++ b/src/core/server/rendering/rendering_service.mock.ts @@ -6,7 +6,14 @@ * Side Public License, v 1. */ -import { InternalRenderingServiceSetup } from './types'; +import { InternalRenderingServicePreboot, InternalRenderingServiceSetup } from './types'; + +function createRenderingPreboot() { + const mocked: jest.Mocked = { + render: jest.fn().mockResolvedValue(''), + }; + return mocked; +} function createRenderingSetup() { const mocked: jest.Mocked = { @@ -16,5 +23,6 @@ function createRenderingSetup() { } export const renderingMock = { + createPrebootContract: createRenderingPreboot, createSetupContract: createRenderingSetup, }; diff --git a/src/core/server/rendering/rendering_service.test.ts b/src/core/server/rendering/rendering_service.test.ts index bba0dc6fd8a67e..de7d21add6c6ca 100644 --- a/src/core/server/rendering/rendering_service.test.ts +++ b/src/core/server/rendering/rendering_service.test.ts @@ -16,9 +16,14 @@ import { import { load } from 'cheerio'; import { httpServerMock } from '../http/http_server.mocks'; +import { mockRouter } from '../http/router/router.mock'; import { uiSettingsServiceMock } from '../ui_settings/ui_settings_service.mock'; -import { mockRenderingServiceParams, mockRenderingSetupDeps } from './__mocks__/params'; -import { InternalRenderingServiceSetup } from './types'; +import { + mockRenderingServiceParams, + mockRenderingPrebootDeps, + mockRenderingSetupDeps, +} from './__mocks__/params'; +import { InternalRenderingServicePreboot, InternalRenderingServiceSetup } from './types'; import { RenderingService } from './rendering_service'; const INJECTED_METADATA = { @@ -43,106 +48,145 @@ const INJECTED_METADATA = { const { createKibanaRequest, createRawRequest } = httpServerMock; -describe('RenderingService', () => { - let service: RenderingService; +function renderTestCases( + getRender: () => Promise< + [ + InternalRenderingServicePreboot['render'] | InternalRenderingServiceSetup['render'], + typeof mockRenderingPrebootDeps | typeof mockRenderingSetupDeps + ] + > +) { + describe('render()', () => { + let uiSettings: ReturnType; + + beforeEach(async () => { + uiSettings = uiSettingsServiceMock.createClient(); + uiSettings.getRegistered.mockReturnValue({ + registered: { name: 'title' }, + }); + }); - beforeEach(() => { - jest.clearAllMocks(); - service = new RenderingService(mockRenderingServiceParams); + it('renders "core" page', async () => { + const [render] = await getRender(); + const content = await render(createKibanaRequest(), uiSettings); + const dom = load(content); + const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); - getSettingValueMock.mockImplementation((settingName: string) => settingName); - getStylesheetPathsMock.mockReturnValue(['/style-1.css', '/style-2.css']); - }); + expect(data).toMatchSnapshot(INJECTED_METADATA); + }); - describe('setup()', () => { - it('calls `registerBootstrapRoute` with the correct parameters', async () => { - await service.setup(mockRenderingSetupDeps); + it('renders "core" page for blank basepath', async () => { + const [render, deps] = await getRender(); + deps.http.basePath.get.mockReturnValueOnce(''); - expect(registerBootstrapRouteMock).toHaveBeenCalledTimes(1); - expect(registerBootstrapRouteMock).toHaveBeenCalledWith({ - router: expect.any(Object), - renderer: bootstrapRendererMock, - }); - }); + const content = await render(createKibanaRequest(), uiSettings); + const dom = load(content); + const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); - describe('render()', () => { - let uiSettings: ReturnType; - let render: InternalRenderingServiceSetup['render']; + expect(data).toMatchSnapshot(INJECTED_METADATA); + }); - beforeEach(async () => { - uiSettings = uiSettingsServiceMock.createClient(); - uiSettings.getRegistered.mockReturnValue({ - registered: { name: 'title' }, - }); - render = (await service.setup(mockRenderingSetupDeps)).render; - }); + it('renders "core" page driven by settings', async () => { + uiSettings.getUserProvided.mockResolvedValue({ 'theme:darkMode': { userValue: true } }); + const [render] = await getRender(); + const content = await render(createKibanaRequest(), uiSettings); + const dom = load(content); + const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); - it('renders "core" page', async () => { - const content = await render(createKibanaRequest(), uiSettings); - const dom = load(content); - const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); + expect(data).toMatchSnapshot(INJECTED_METADATA); + }); - expect(data).toMatchSnapshot(INJECTED_METADATA); + it('renders "core" with excluded user settings', async () => { + const [render] = await getRender(); + const content = await render(createKibanaRequest(), uiSettings, { + includeUserSettings: false, }); + const dom = load(content); + const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); - it('renders "core" page for blank basepath', async () => { - mockRenderingSetupDeps.http.basePath.get.mockReturnValueOnce(''); + expect(data).toMatchSnapshot(INJECTED_METADATA); + }); - const content = await render(createKibanaRequest(), uiSettings); - const dom = load(content); - const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); + it('renders "core" from legacy request', async () => { + const [render] = await getRender(); + const content = await render(createRawRequest(), uiSettings); + const dom = load(content); + const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); - expect(data).toMatchSnapshot(INJECTED_METADATA); + expect(data).toMatchSnapshot(INJECTED_METADATA); + }); + + it('calls `getStylesheetPaths` with the correct parameters', async () => { + getSettingValueMock.mockImplementation((settingName: string) => { + if (settingName === 'theme:darkMode') { + return true; + } + if (settingName === 'theme:version') { + return 'v8'; + } + return settingName; }); - it('renders "core" page driven by settings', async () => { - uiSettings.getUserProvided.mockResolvedValue({ 'theme:darkMode': { userValue: true } }); - const content = await render(createKibanaRequest(), uiSettings); - const dom = load(content); - const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); + const [render] = await getRender(); + await render(createKibanaRequest(), uiSettings); - expect(data).toMatchSnapshot(INJECTED_METADATA); + expect(getStylesheetPathsMock).toHaveBeenCalledTimes(1); + expect(getStylesheetPathsMock).toHaveBeenCalledWith({ + darkMode: true, + themeVersion: 'v8', + basePath: '/mock-server-basepath', + buildNum: expect.any(Number), }); + }); + }); +} - it('renders "core" with excluded user settings', async () => { - const content = await render(createKibanaRequest(), uiSettings, { - includeUserSettings: false, - }); - const dom = load(content); - const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); +describe('RenderingService', () => { + let service: RenderingService; - expect(data).toMatchSnapshot(INJECTED_METADATA); - }); + beforeEach(() => { + jest.clearAllMocks(); + service = new RenderingService(mockRenderingServiceParams); - it('renders "core" from legacy request', async () => { - const content = await render(createRawRequest(), uiSettings); - const dom = load(content); - const data = JSON.parse(dom('kbn-injected-metadata').attr('data') ?? '""'); + getSettingValueMock.mockImplementation((settingName: string) => settingName); + getStylesheetPathsMock.mockReturnValue(['/style-1.css', '/style-2.css']); + }); + + describe('preboot()', () => { + it('calls `registerBootstrapRoute` with the correct parameters', async () => { + const routerMock = mockRouter.create(); + mockRenderingPrebootDeps.http.registerRoutes.mockImplementation((path, callback) => + callback(routerMock) + ); + + await service.preboot(mockRenderingPrebootDeps); - expect(data).toMatchSnapshot(INJECTED_METADATA); + expect(registerBootstrapRouteMock).toHaveBeenCalledTimes(1); + expect(registerBootstrapRouteMock).toHaveBeenCalledWith({ + router: routerMock, + renderer: bootstrapRendererMock, }); + }); + + renderTestCases(async () => { + return [(await service.preboot(mockRenderingPrebootDeps)).render, mockRenderingPrebootDeps]; + }); + }); + + describe('setup()', () => { + it('calls `registerBootstrapRoute` with the correct parameters', async () => { + await service.setup(mockRenderingSetupDeps); - it('calls `getStylesheetPaths` with the correct parameters', async () => { - getSettingValueMock.mockImplementation((settingName: string) => { - if (settingName === 'theme:darkMode') { - return true; - } - if (settingName === 'theme:version') { - return 'v8'; - } - return settingName; - }); - - await render(createKibanaRequest(), uiSettings); - - expect(getStylesheetPathsMock).toHaveBeenCalledTimes(1); - expect(getStylesheetPathsMock).toHaveBeenCalledWith({ - darkMode: true, - themeVersion: 'v8', - basePath: '/mock-server-basepath', - buildNum: expect.any(Number), - }); + expect(registerBootstrapRouteMock).toHaveBeenCalledTimes(1); + expect(registerBootstrapRouteMock).toHaveBeenCalledWith({ + router: expect.any(Object), + renderer: bootstrapRendererMock, }); }); + + renderTestCases(async () => { + await service.preboot(mockRenderingPrebootDeps); + return [(await service.setup(mockRenderingSetupDeps)).render, mockRenderingSetupDeps]; + }); }); }); diff --git a/src/core/server/rendering/rendering_service.tsx b/src/core/server/rendering/rendering_service.tsx index fd4e1140d68b49..2d95822d922192 100644 --- a/src/core/server/rendering/rendering_service.tsx +++ b/src/core/server/rendering/rendering_service.tsx @@ -16,100 +16,130 @@ import { CoreContext } from '../core_context'; import { Template } from './views'; import { IRenderOptions, + RenderingPrebootDeps, RenderingSetupDeps, + InternalRenderingServicePreboot, InternalRenderingServiceSetup, RenderingMetadata, } from './types'; import { registerBootstrapRoute, bootstrapRendererFactory } from './bootstrap'; import { getSettingValue, getStylesheetPaths } from './render_utils'; +import { KibanaRequest, LegacyRequest } from '../http'; +import { IUiSettingsClient } from '../ui_settings'; + +type RenderOptions = (RenderingPrebootDeps & { status?: never }) | RenderingSetupDeps; /** @internal */ export class RenderingService { constructor(private readonly coreContext: CoreContext) {} + public async preboot({ + http, + uiPlugins, + }: RenderingPrebootDeps): Promise { + http.registerRoutes('', (router) => { + registerBootstrapRoute({ + router, + renderer: bootstrapRendererFactory({ + uiPlugins, + serverBasePath: http.basePath.serverBasePath, + packageInfo: this.coreContext.env.packageInfo, + auth: http.auth, + }), + }); + }); + + return { + render: this.render.bind(this, { http, uiPlugins }), + }; + } + public async setup({ http, status, uiPlugins, }: RenderingSetupDeps): Promise { - const router = http.createRouter(''); - - const bootstrapRenderer = bootstrapRendererFactory({ - uiPlugins, - serverBasePath: http.basePath.serverBasePath, - packageInfo: this.coreContext.env.packageInfo, - auth: http.auth, + registerBootstrapRoute({ + router: http.createRouter(''), + renderer: bootstrapRendererFactory({ + uiPlugins, + serverBasePath: http.basePath.serverBasePath, + packageInfo: this.coreContext.env.packageInfo, + auth: http.auth, + }), }); - registerBootstrapRoute({ router, renderer: bootstrapRenderer }); return { - render: async ( - request, - uiSettings, - { includeUserSettings = true, vars }: IRenderOptions = {} - ) => { - const env = { - mode: this.coreContext.env.mode, - packageInfo: this.coreContext.env.packageInfo, - }; - const buildNum = env.packageInfo.buildNum; - const basePath = http.basePath.get(request); - const { serverBasePath, publicBaseUrl } = http.basePath; - const settings = { - defaults: uiSettings.getRegistered(), - user: includeUserSettings ? await uiSettings.getUserProvided() : {}, - }; + render: this.render.bind(this, { http, uiPlugins, status }), + }; + } - const darkMode = getSettingValue('theme:darkMode', settings, Boolean); - const themeVersion = getSettingValue('theme:version', settings, String); + private async render( + { http, uiPlugins, status }: RenderOptions, + request: KibanaRequest | LegacyRequest, + uiSettings: IUiSettingsClient, + { includeUserSettings = true, vars }: IRenderOptions = {} + ) { + const env = { + mode: this.coreContext.env.mode, + packageInfo: this.coreContext.env.packageInfo, + }; + const buildNum = env.packageInfo.buildNum; + const basePath = http.basePath.get(request); + const { serverBasePath, publicBaseUrl } = http.basePath; + const settings = { + defaults: uiSettings.getRegistered() ?? {}, + user: includeUserSettings ? await uiSettings.getUserProvided() : {}, + }; - const stylesheetPaths = getStylesheetPaths({ - darkMode, - themeVersion, - basePath: serverBasePath, - buildNum, - }); + const darkMode = getSettingValue('theme:darkMode', settings, Boolean); + const themeVersion = getSettingValue('theme:version', settings, String); - const metadata: RenderingMetadata = { - strictCsp: http.csp.strict, - uiPublicUrl: `${basePath}/ui`, - bootstrapScriptUrl: `${basePath}/bootstrap.js`, - i18n: i18n.translate, - locale: i18n.getLocale(), - darkMode, - stylesheetPaths, - themeVersion, - injectedMetadata: { - version: env.packageInfo.version, - buildNumber: env.packageInfo.buildNum, - branch: env.packageInfo.branch, - basePath, - serverBasePath, - publicBaseUrl, - env, - anonymousStatusPage: status.isStatusPageAnonymous(), - i18n: { - translationsUrl: `${basePath}/translations/${i18n.getLocale()}.json`, - }, - csp: { warnLegacyBrowsers: http.csp.warnLegacyBrowsers }, - externalUrl: http.externalUrl, - vars: vars ?? {}, - uiPlugins: await Promise.all( - [...uiPlugins.public].map(async ([id, plugin]) => ({ - id, - plugin, - config: await getUiConfig(uiPlugins, id), - })) - ), - legacyMetadata: { - uiSettings: settings, - }, - }, - }; + const stylesheetPaths = getStylesheetPaths({ + darkMode, + themeVersion, + basePath: serverBasePath, + buildNum, + }); - return `${renderToStaticMarkup(