From 4c8823761193151c56c8835b05c1c8b375ec9f6e Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 27 Aug 2020 16:09:12 -0700 Subject: [PATCH 01/40] Adds new elasticsearch client to telemetry plugin --- .../telemetry_plugin_collector.ts | 6 +++++- src/plugins/telemetry/server/plugin.ts | 17 ++++++++++------- .../telemetry_collection/register_collection.ts | 6 ++++-- .../server/plugin.ts | 5 +++++ .../server/types.ts | 10 +++++++++- .../server/collector/collector.ts | 4 ++-- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts b/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts index 05836b8448a688..0a33fb7a5f8ec5 100644 --- a/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts +++ b/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts @@ -19,7 +19,11 @@ import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { ISavedObjectsRepository, SavedObjectsClient } from '../../../../../core/server'; +import { + ISavedObjectsRepository, + SavedObjectsClient, + IClusterClient, +} from '../../../../../core/server'; import { getTelemetrySavedObject, TelemetrySavedObject } from '../../telemetry_repository'; import { getTelemetryOptIn, getTelemetrySendUsageFrom } from '../../../common/telemetry_config'; import { UsageCollectionSetup } from '../../../../usage_collection/server'; diff --git a/src/plugins/telemetry/server/plugin.ts b/src/plugins/telemetry/server/plugin.ts index 005c5f96d98d03..a0bd3664209811 100644 --- a/src/plugins/telemetry/server/plugin.ts +++ b/src/plugins/telemetry/server/plugin.ts @@ -34,6 +34,7 @@ import { SavedObjectsClient, Plugin, Logger, + IClusterClient, } from '../../../core/server'; import { registerRoutes } from './routes'; import { registerCollection } from './telemetry_collection'; @@ -83,6 +84,7 @@ export class TelemetryPlugin implements Plugin) { this.logger = initializerContext.logger.get(); @@ -102,8 +104,11 @@ export class TelemetryPlugin implements Plugin this.elasticsearchClient + ); const router = http.createRouter(); registerRoutes({ @@ -126,14 +131,12 @@ export class TelemetryPlugin implements Plugin { - const { savedObjects, uiSettings } = core; + public async start(core: CoreStart, { telemetryCollectionManager }: TelemetryPluginsStart) { + const { savedObjects, uiSettings, elasticsearch } = core; this.savedObjectsClient = savedObjects.createInternalRepository(); const savedObjectsClient = new SavedObjectsClient(this.savedObjectsClient); this.uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient); + this.elasticsearchClient = elasticsearch.client; try { await handleOldSettings(savedObjectsClient, this.uiSettingsClient); diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts index 438fcadad92555..f3148a014a61aa 100644 --- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts +++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts @@ -36,7 +36,7 @@ * under the License. */ -import { ILegacyClusterClient } from 'kibana/server'; +import { ILegacyClusterClient, IClusterClient } from 'kibana/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { getLocalStats } from './get_local_stats'; import { getClusterUuids } from './get_cluster_stats'; @@ -44,10 +44,12 @@ import { getLocalLicense } from './get_local_license'; export function registerCollection( telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, - esCluster: ILegacyClusterClient + esCluster: ILegacyClusterClient, + esClientGetter: () => IClusterClient | undefined ) { telemetryCollectionManager.setCollection({ esCluster, + esClientGetter, title: 'local', priority: 0, statsGetter: getLocalStats, diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 051bb3a11cb16c..92ead9fc87153c 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -86,6 +86,7 @@ export class TelemetryCollectionManagerPlugin title, priority, esCluster, + esClientGetter, statsGetter, clusterDetailsGetter, licenseGetter, @@ -105,6 +106,9 @@ export class TelemetryCollectionManagerPlugin if (!esCluster) { throw Error('esCluster name must be set for the getCluster method.'); } + if (!esClientGetter) { + throw Error('esClient getter method not set.'); + } if (!clusterDetailsGetter) { throw Error('Cluster UUIds method is not set.'); } @@ -118,6 +122,7 @@ export class TelemetryCollectionManagerPlugin clusterDetailsGetter, esCluster, title, + esClientGetter, }); this.usageGetterMethodPriority = priority; } diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index 16f96c07fd8ead..ff7e165215e322 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -17,7 +17,13 @@ * under the License. */ -import { LegacyAPICaller, Logger, KibanaRequest, ILegacyClusterClient } from 'kibana/server'; +import { + LegacyAPICaller, + Logger, + KibanaRequest, + ILegacyClusterClient, + IClusterClient, +} from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { TelemetryCollectionManagerPlugin } from './plugin'; @@ -130,6 +136,7 @@ export interface CollectionConfig< title: string; priority: number; esCluster: ILegacyClusterClient; + esClientGetter: () => IClusterClient | undefined; statsGetter: StatsGetter; clusterDetailsGetter: ClusterDetailsGetter; licenseGetter: LicenseGetter; @@ -145,5 +152,6 @@ export interface Collection< licenseGetter: LicenseGetter; clusterDetailsGetter: ClusterDetailsGetter; esCluster: ILegacyClusterClient; + esClientGetter: () => IClusterClient | undefined; title: string; } diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index d57700024c088f..3895769fbb4be9 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Logger, LegacyAPICaller } from 'kibana/server'; +import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; export type CollectorFormatForBulkUpload = (result: T) => { type: string; payload: U }; @@ -48,7 +48,7 @@ export interface CollectorOptions { type: string; init?: Function; schema?: MakeSchemaFrom>; // Using Required to enforce all optional keys in the object - fetch: (callCluster: LegacyAPICaller) => Promise | T; + fetch: (callCluster: LegacyAPICaller, esClient: ElasticsearchClient) => Promise | T; /* * A hook for allowing the fetched data payload to be organized into a typed * data model for internal bulk upload. See defaultFormatterForBulkUpload for From 57dfd55c883ccaf8b81dcd42fc34d0e579cae80f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 27 Aug 2020 16:42:22 -0700 Subject: [PATCH 02/40] Adds esClient getter to local_xpack and monitoring setCollection --- .../telemetry_collection_manager/server/plugin.ts | 3 +++ x-pack/plugins/monitoring/server/plugin.ts | 13 ++++++++++--- .../register_monitoring_collection.ts | 4 +++- .../telemetry_collection_xpack/server/plugin.ts | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 92ead9fc87153c..48ffdf17eaa26f 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -107,6 +107,9 @@ export class TelemetryCollectionManagerPlugin throw Error('esCluster name must be set for the getCluster method.'); } if (!esClientGetter) { + this.logger.warn( + `esClient getter method returns ${esClientGetter} for ${title} collection.` + ); throw Error('esClient getter method not set.'); } if (!clusterDetailsGetter) { diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index f5cbadb523a819..3d9dfd08b9f9be 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -20,6 +20,7 @@ import { CoreStart, CustomHttpResponseOptions, ResponseError, + IClusterClient, } from 'kibana/server'; import { LOGGING_TAG, @@ -71,6 +72,7 @@ export class Plugin { private monitoringCore = {} as MonitoringCore; private legacyShimDependencies = {} as LegacyShimDependencies; private bulkUploader: IBulkUploader = {} as IBulkUploader; + private elasticsearchClient: IClusterClient; constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; @@ -141,9 +143,14 @@ export class Plugin { // Initialize telemetry if (plugins.telemetryCollectionManager) { - registerMonitoringCollection(plugins.telemetryCollectionManager, this.cluster, { - maxBucketSize: config.ui.max_bucket_size, - }); + registerMonitoringCollection( + plugins.telemetryCollectionManager, + this.cluster, + () => this.elasticsearchClient, + { + maxBucketSize: config.ui.max_bucket_size, + } + ); } // Register collector objects for stats to show up in the APIs diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts index 1e5549fe4d8003..3648ae4bd8551b 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ILegacyCustomClusterClient } from 'kibana/server'; +import { ILegacyCustomClusterClient, IClusterClient } from 'kibana/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { getAllStats, CustomContext } from './get_all_stats'; import { getClusterUuids } from './get_cluster_uuids'; @@ -13,10 +13,12 @@ import { getLicenses } from './get_licenses'; export function registerMonitoringCollection( telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, esCluster: ILegacyCustomClusterClient, + esClientGetter: () => IClusterClient | undefined, customContext: CustomContext ) { telemetryCollectionManager.setCollection({ esCluster, + esClientGetter, title: 'monitoring', priority: 2, statsGetter: getAllStats, diff --git a/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts b/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts index 3f01d7423eded1..6ef44e325b0a70 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts @@ -4,7 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { PluginInitializerContext, CoreSetup, CoreStart, Plugin } from 'kibana/server'; +import { + PluginInitializerContext, + CoreSetup, + CoreStart, + Plugin, + IClusterClient, +} from 'kibana/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { getClusterUuids, getLocalLicense } from '../../../../src/plugins/telemetry/server'; import { getStatsWithXpack } from './telemetry_collection'; @@ -14,11 +20,13 @@ interface TelemetryCollectionXpackDepsSetup { } export class TelemetryCollectionXpackPlugin implements Plugin { + private elasticsearchClient?: IClusterClient; constructor(initializerContext: PluginInitializerContext) {} public setup(core: CoreSetup, { telemetryCollectionManager }: TelemetryCollectionXpackDepsSetup) { telemetryCollectionManager.setCollection({ esCluster: core.elasticsearch.legacy.client, + esClientGetter: () => this.elasticsearchClient, title: 'local_xpack', priority: 1, statsGetter: getStatsWithXpack, @@ -27,5 +35,7 @@ export class TelemetryCollectionXpackPlugin implements Plugin { }); } - public start(core: CoreStart) {} + public start(core: CoreStart) { + this.elasticsearchClient = core.elasticsearch.client; + } } From 844fc51e77899f2f32e997f21dc345f0606c522e Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 27 Aug 2020 17:12:13 -0700 Subject: [PATCH 03/40] Provides the new es client to the stats collection config --- .../server/plugin.ts | 24 +++++++++++++++---- .../server/types.ts | 2 ++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 48ffdf17eaa26f..ee9a1a30bb1ca8 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -24,6 +24,7 @@ import { CoreStart, Plugin, Logger, + IClusterClient, } from '../../../core/server'; import { @@ -107,10 +108,7 @@ export class TelemetryCollectionManagerPlugin throw Error('esCluster name must be set for the getCluster method.'); } if (!esClientGetter) { - this.logger.warn( - `esClient getter method returns ${esClientGetter} for ${title} collection.` - ); - throw Error('esClient getter method not set.'); + throw Error('esClientGetter metod not set.'); } if (!clusterDetailsGetter) { throw Error('Cluster UUIds method is not set.'); @@ -141,8 +139,24 @@ export class TelemetryCollectionManagerPlugin const callCluster = config.unencrypted ? collection.esCluster.asScoped(request).callAsCurrentUser : collection.esCluster.callAsInternalUser; + // Scope the new elasticsearch Client appropriately and pass to the stats collection config + // Question: Do we need to add it? Now we have two clients, the legacy and new client. + const esClient = this.getScopedEsClientIfNeeded(collection.esClientGetter, config); + + return { callCluster, start, end, usageCollection, esClient }; + } - return { callCluster, start, end, usageCollection }; + // not sure if we need to use the new client to get the stats collection config just yet. + private getScopedEsClientIfNeeded( + esClientGetter: () => IClusterClient | undefined, + config: StatsGetterConfig + ) { + if (esClientGetter() !== undefined) { + return config.unencrypted + ? esClientGetter()!.asScoped(config.request).asCurrentUser + : esClientGetter()!.asInternalUser; + } + return undefined; } private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) { diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index ff7e165215e322..0dd17c7d07eadf 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -23,6 +23,7 @@ import { KibanaRequest, ILegacyClusterClient, IClusterClient, + ElasticsearchClient, } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { TelemetryCollectionManagerPlugin } from './plugin'; @@ -73,6 +74,7 @@ export interface StatsCollectionConfig { callCluster: LegacyAPICaller; start: string | number; end: string | number; + esClient?: ElasticsearchClient; } export interface BasicStatsPayload { From 21f326133fbccc961aaa3143bcf31d5f849bba23 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 27 Aug 2020 18:49:06 -0700 Subject: [PATCH 04/40] Adds asInternalUser to bulkFetch --- .../server/telemetry_collection/get_kibana.ts | 7 ++- .../server/plugin.ts | 63 +++++++++---------- .../server/types.ts | 1 + src/plugins/usage_collection/README.md | 2 +- .../server/collector/collector_set.ts | 6 +- 5 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts index 5d27774a630a59..156aee3a0988f4 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts @@ -19,7 +19,7 @@ import { omit } from 'lodash'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { StatsCollectionContext } from 'src/plugins/telemetry_collection_manager/server'; export interface KibanaUsageStats { @@ -83,8 +83,9 @@ export function handleKibanaStats( export async function getKibana( usageCollection: UsageCollectionSetup, - callWithInternalUser: LegacyAPICaller + callWithInternalUser: LegacyAPICaller, + asInternalUser: ElasticsearchClient ): Promise { - const usage = await usageCollection.bulkFetch(callWithInternalUser); + const usage = await usageCollection.bulkFetch(callWithInternalUser, asInternalUser); return usageCollection.toObject(usage); } diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index ee9a1a30bb1ca8..3e808401c2084e 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -140,25 +140,14 @@ export class TelemetryCollectionManagerPlugin ? collection.esCluster.asScoped(request).callAsCurrentUser : collection.esCluster.callAsInternalUser; // Scope the new elasticsearch Client appropriately and pass to the stats collection config - // Question: Do we need to add it? Now we have two clients, the legacy and new client. - const esClient = this.getScopedEsClientIfNeeded(collection.esClientGetter, config); + // Problem: I need a Scopeable request to pass to asScoped. + const esClient = config.unencrypted + ? collection.esClientGetter()!.asScoped(config.request).asCurrentUser + : collection.esClientGetter()!.asInternalUser; return { callCluster, start, end, usageCollection, esClient }; } - // not sure if we need to use the new client to get the stats collection config just yet. - private getScopedEsClientIfNeeded( - esClientGetter: () => IClusterClient | undefined, - config: StatsGetterConfig - ) { - if (esClientGetter() !== undefined) { - return config.unencrypted - ? esClientGetter()!.asScoped(config.request).asCurrentUser - : esClientGetter()!.asInternalUser; - } - return undefined; - } - private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) { if (!this.usageCollection) { return []; @@ -213,29 +202,33 @@ export class TelemetryCollectionManagerPlugin if (!this.usageCollection) { return []; } + // call the esClientGetter and check if it returns undefined. + for (const collection of this.collections) { - const statsCollectionConfig = this.getStatsCollectionConfig( - config, - collection, - this.usageCollection - ); - try { - const usageData = await this.getUsageForCollection(collection, statsCollectionConfig); - if (usageData.length) { - this.logger.debug(`Got Usage using ${collection.title} collection.`); - if (config.unencrypted) { - return usageData; - } + if (collection.esClientGetter() !== undefined) { + const statsCollectionConfig = this.getStatsCollectionConfig( + config, + collection, + this.usageCollection + ); + try { + const usageData = await this.getUsageForCollection(collection, statsCollectionConfig); + if (usageData.length) { + this.logger.debug(`Got Usage using ${collection.title} collection.`); + if (config.unencrypted) { + return usageData; + } - return encryptTelemetry(usageData.filter(isClusterOptedIn), { - useProdKey: this.isDistributable, - }); + return encryptTelemetry(usageData.filter(isClusterOptedIn), { + useProdKey: this.isDistributable, + }); + } + } catch (err) { + this.logger.debug( + `Failed to collect any usage with registered collection ${collection.title}.` + ); + // swallow error to try next collection; } - } catch (err) { - this.logger.debug( - `Failed to collect any usage with registered collection ${collection.title}.` - ); - // swallow error to try next collection; } } diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index 0dd17c7d07eadf..2b6ca0314d9cd7 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -24,6 +24,7 @@ import { ILegacyClusterClient, IClusterClient, ElasticsearchClient, + ScopeableRequest, } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { TelemetryCollectionManagerPlugin } from './plugin'; diff --git a/src/plugins/usage_collection/README.md b/src/plugins/usage_collection/README.md index 0b1cca07de007d..fd8b8c0aa83c7f 100644 --- a/src/plugins/usage_collection/README.md +++ b/src/plugins/usage_collection/README.md @@ -63,7 +63,7 @@ All you need to provide is a `type` for organizing your fields, `schema` field t total: 'long', }, }, - fetch: async (callCluster: APICluster) => { + fetch: async (callCluster: APICluster, esClient: IClusterClient) => { // query ES and get some data // summarize the data into a model diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index fce17a46b71689..29a9521bcb8d88 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -18,7 +18,7 @@ */ import { snakeCase } from 'lodash'; -import { Logger, LegacyAPICaller } from 'kibana/server'; +import { Logger, LegacyAPICaller, IClusterClient, ElasticsearchClient } from 'kibana/server'; import { Collector, CollectorOptions } from './collector'; import { UsageCollector } from './usage_collector'; @@ -117,8 +117,10 @@ export class CollectorSet { return allReady; }; + // all collections eventually pass through bulkFetch. public bulkFetch = async ( callCluster: LegacyAPICaller, + esClient: ElasticsearchClient, collectors: Map> = this.collectors ) => { const responses = await Promise.all( @@ -127,7 +129,7 @@ export class CollectorSet { try { return { type: collector.type, - result: await collector.fetch(callCluster), + result: await collector.fetch(callCluster, esClient), }; } catch (err) { this.logger.warn(err); From dbaa02e58de611cc9a3e9a6180c3db8c085022f7 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 27 Aug 2020 18:51:18 -0700 Subject: [PATCH 05/40] TODO: add scopedEsClient to remaining bulkFetch calls --- .../monitoring/server/kibana_monitoring/bulk_uploader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index 5d8af8d71b7fce..88147acd84fadc 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -136,7 +136,7 @@ export class BulkUploader { } return; } - + // TINA TODO: add the new client call in here. const data = await usageCollection.bulkFetch(this._cluster.callAsInternalUser); const payload = this.toBulkUploadFormat(compact(data), usageCollection); if (payload && payload.length > 0) { From 50e2d1fed8ba12d99e6df07a3668ec3db6eb4b3f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 31 Aug 2020 16:23:47 -0700 Subject: [PATCH 06/40] Adds esClient to all internal getters, adds hardcoded useLegacy boolean for now --- .../telemetry_collection/get_cluster_info.ts | 10 ++- .../telemetry_collection/get_cluster_stats.ts | 17 +++-- .../get_data_telemetry/get_data_telemetry.ts | 72 ++++++++++++++++++- .../telemetry_collection/get_local_stats.ts | 14 ++-- .../telemetry_collection/get_nodes_usage.ts | 18 +++-- .../register_collection.ts | 1 + .../server/plugin.ts | 14 ++-- .../server/types.ts | 5 +- .../server/collector/collector_set.ts | 7 +- 9 files changed, 123 insertions(+), 35 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 4a33356ee97614..8508bdcb6d4ba2 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; // This can be removed when the ES client improves the types export interface ESClusterInfo { @@ -42,7 +42,11 @@ export interface ESClusterInfo { * This is the equivalent to GET / * * @param {function} callCluster The callWithInternalUser handler (exposed for testing) + * @param {function} esClient The asInternalUser handler (exposed for testing) + * + * TODO: needs work on using the new client */ -export function getClusterInfo(callCluster: LegacyAPICaller) { - return callCluster('info'); +export function getClusterInfo(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { + const useLegacy = true; + return useLegacy ? callCluster('info') : esClient.info; } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index d7c0110a99c6fd..ba226c5d63b32f 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -18,23 +18,30 @@ */ import { ClusterDetailsGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { TIMEOUT } from './constants'; /** * Get the cluster stats from the connected cluster. * * This is the equivalent to GET /_cluster/stats?timeout=30s. */ -export async function getClusterStats(callCluster: LegacyAPICaller) { - return await callCluster('cluster.stats', { +export async function getClusterStats(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { + // add some sort of a check to see which client we want or, + // hack: + const useLegacy = true; + // alternatively, just replace the call with the new client + const legacyClusterStats = await callCluster('cluster.stats', { timeout: TIMEOUT, }); + // new client implementation: + const { body } = await esClient.cluster.stats({ timeout: TIMEOUT }); // the response has changed + return useLegacy ? legacyClusterStats : body; } /** * Get the cluster uuids from the connected cluster. */ -export const getClusterUuids: ClusterDetailsGetter = async ({ callCluster }) => { - const result = await getClusterStats(callCluster); +export const getClusterUuids: ClusterDetailsGetter = async ({ callCluster, esClient }) => { + const result = await getClusterStats(callCluster, esClient); return [{ clusterUuid: result.cluster_uuid }]; }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index f4734dde251cc8..97574102432835 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE, DataPatternName, @@ -187,6 +187,19 @@ interface IndexStats { }; } +interface IndexStatistics { + [indexName: string]: { + total: { + docs: { + count: number; + deleted: number; + }; + store: { + size_in_bytes: number; + }; + }; + }; +} interface IndexMappings { [indexName: string]: { mappings: { @@ -223,13 +236,68 @@ interface IndexMappings { }; }; } +interface GetMappingsResponse { + body: IndexMappings; + statusCode: number | null; + headers: Record | null; + warnings: string[] | null; + meta: object; +} +interface IndexStatsResponse { + body: IndexStatistics; + statusCode: number | null; + headers: Record | null; + warnings: string[] | null; + meta: object; +} -export async function getDataTelemetry(callCluster: LegacyAPICaller) { +export async function getDataTelemetry( + callCluster: LegacyAPICaller, + esClient: ElasticsearchClient +) { try { const index = [ ...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern), '*-*-*', // Include data-streams aliases `{type}-{dataset}-{namespace}` ]; + // check which client we need to use for now, evetually replace the legacy client with the new one. Hard-coding use of the legacy client for now + const useLegacy = true; + if (!useLegacy) { + const [getMappingsResponse, indexStatsResponse]: [ + GetMappingsResponse, + IndexStatsResponse + ] = await Promise.all([ + // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value + esClient.indices.getMapping({ + index: '*', + filter_path: [ + // _meta.beat tells the shipper + '*.mappings._meta.beat', + // _meta.package.name tells the Ingest Manager's package + '*.mappings._meta.package.name', + // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it + '*.mappings._meta.managed_by', + // Does it have `ecs.version` in the mappings? => It follows the ECS conventions + '*.mappings.properties.ecs.properties.version.type', + + // If `data_stream.type` is a `constant_keyword`, it can be reported as a type + '*.mappings.properties.data_stream.properties.type.value', + // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset + '*.mappings.properties.data_stream.properties.dataset.value', + ], + }), + // GET /_stats/docs,store?level=indices&filter_path=indices.*.total + esClient.indices.stats({ + index, + level: 'indices', + metric: ['docs', 'store'], + filter_path: ['indices.*.total'], + }), + ]); + const indexMappings: IndexMappings = getMappingsResponse.body; + const indexStats: IndexStats = indexStatsResponse.body; + } + const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([ // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value callCluster('indices.getMapping', { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index 98c83a3394628b..8707b43fa26c96 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -65,19 +65,19 @@ export type TelemetryLocalStats = ReturnType; */ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( clustersDetails, - config, + config, // contains the new esClient already scoped context ) => { - const { callCluster, usageCollection } = config; + const { callCluster, usageCollection, esClient } = config; return await Promise.all( clustersDetails.map(async (clustersDetail) => { const [clusterInfo, clusterStats, nodesUsage, kibana, dataTelemetry] = await Promise.all([ - getClusterInfo(callCluster), // cluster info - getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_) - getNodesUsage(callCluster), // nodes_usage info - getKibana(usageCollection, callCluster), - getDataTelemetry(callCluster), + getClusterInfo(callCluster, esClient), // cluster info -> careful here: the new ESClient + getClusterStats(callCluster, esClient), // cluster stats (not to be confused with cluster _state_) + getNodesUsage(callCluster, esClient), // nodes_usage info + getKibana(usageCollection, callCluster, esClient), + getDataTelemetry(callCluster, esClient), ]); return handleLocalStats( clusterInfo, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index c5c110fbb4149b..a977d9fb57b470 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { TIMEOUT } from './constants'; export interface NodeAggregation { @@ -44,7 +44,8 @@ export interface NodesFeatureUsageResponse { } export type NodesUsageGetter = ( - callCluster: LegacyAPICaller + callCluster: LegacyAPICaller, + esClient: ElasticsearchClient ) => Promise<{ nodes: NodeObj[] | Array<{}> }>; /** * Get the nodes usage data from the connected cluster. @@ -54,16 +55,19 @@ export type NodesUsageGetter = ( * The Nodes usage API was introduced in v6.0.0 */ export async function fetchNodesUsage( - callCluster: LegacyAPICaller + callCluster: LegacyAPICaller, + esClient: ElasticsearchClient ): Promise { - const response = await callCluster('transport.request', { + const useLegacy = true; + const legacyResponse = await callCluster('transport.request', { method: 'GET', path: '/_nodes/usage', query: { timeout: TIMEOUT, }, }); - return response; + const { body } = await esClient.nodes.usage({ metric: '_all', timeout: TIMEOUT }); + return useLegacy ? legacyResponse : body; } /** @@ -71,8 +75,8 @@ export async function fetchNodesUsage( * @param callCluster APICaller * @returns Object containing array of modified usage information with the node_id nested within the data for that node. */ -export const getNodesUsage: NodesUsageGetter = async (callCluster) => { - const result = await fetchNodesUsage(callCluster); +export const getNodesUsage: NodesUsageGetter = async (callCluster, esClient) => { + const result = await fetchNodesUsage(callCluster, esClient); const transformedNodes = Object.entries(result?.nodes || {}).map(([key, value]) => ({ ...(value as NodeObj), node_id: key, diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts index f3148a014a61aa..5afa0df71c5cc8 100644 --- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts +++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts @@ -47,6 +47,7 @@ export function registerCollection( esCluster: ILegacyClusterClient, esClientGetter: () => IClusterClient | undefined ) { + // use the new ES client internally for clusterDetailsGetter, licenceGetter and, potentially, statsGetter. telemetryCollectionManager.setCollection({ esCluster, esClientGetter, diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 3e808401c2084e..d25bb3bb8215fe 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -202,9 +202,11 @@ export class TelemetryCollectionManagerPlugin if (!this.usageCollection) { return []; } - // call the esClientGetter and check if it returns undefined. + // This is right before we loop through the collectors to call their fetch methods. + // before looping through each collector and calling its fetch method, we ensure that the esClientGetter returns something, if it doesn't, we skip the collection set (local, xpack_local, monitoring). for (const collection of this.collections) { + // looping through each of the three collections options we have (grouping of usage collection) if (collection.esClientGetter() !== undefined) { const statsCollectionConfig = this.getStatsCollectionConfig( config, @@ -236,15 +238,15 @@ export class TelemetryCollectionManagerPlugin } private async getUsageForCollection( - collection: Collection, - statsCollectionConfig: StatsCollectionConfig + collection: Collection, // contains the esClientGetter method + statsCollectionConfig: StatsCollectionConfig // contains the already-scoped esClient ): Promise { const context: StatsCollectionContext = { logger: this.logger.get(collection.title), version: this.version, ...collection.customContext, }; - + // added call to use new esClient const clustersDetails = await collection.clusterDetailsGetter(statsCollectionConfig, context); if (clustersDetails.length === 0) { @@ -253,8 +255,8 @@ export class TelemetryCollectionManagerPlugin } const [stats, licenses] = await Promise.all([ - collection.statsGetter(clustersDetails, statsCollectionConfig, context), - collection.licenseGetter(clustersDetails, statsCollectionConfig, context), + collection.statsGetter(clustersDetails, statsCollectionConfig, context), // todo: use new esClient + collection.licenseGetter(clustersDetails, statsCollectionConfig, context), // todo: use new ESClient ]); return stats.map((stat) => { diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index 2b6ca0314d9cd7..e155ed17b079b6 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -24,7 +24,6 @@ import { ILegacyClusterClient, IClusterClient, ElasticsearchClient, - ScopeableRequest, } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { TelemetryCollectionManagerPlugin } from './plugin'; @@ -75,7 +74,7 @@ export interface StatsCollectionConfig { callCluster: LegacyAPICaller; start: string | number; end: string | number; - esClient?: ElasticsearchClient; + esClient: ElasticsearchClient; } export interface BasicStatsPayload { @@ -132,6 +131,8 @@ export type LicenseGetter = {}> = ( context: StatsCollectionContext & CustomContext ) => Promise<{ [clusterUuid: string]: ESLicense | undefined }>; +// Note that CollectionConfig is different to Collection!: CollectionConfig ~= Collection & priority. +// This is daft! export interface CollectionConfig< CustomContext extends Record = {}, T extends BasicStatsPayload = BasicStatsPayload diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 29a9521bcb8d88..77c5036a169c8c 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -118,6 +118,7 @@ export class CollectorSet { }; // all collections eventually pass through bulkFetch. + // teh shape of the reponse is different when using the new ES client as is the error handling. public bulkFetch = async ( callCluster: LegacyAPICaller, esClient: ElasticsearchClient, @@ -129,7 +130,7 @@ export class CollectorSet { try { return { type: collector.type, - result: await collector.fetch(callCluster, esClient), + result: await collector.fetch(callCluster, esClient), // note that the response shape is different between hte legacy and new clients. Each `fetch` method will need to ensure it handles reshaping the data correctly. }; } catch (err) { this.logger.warn(err); @@ -151,9 +152,9 @@ export class CollectorSet { return this.makeCollectorSetFromArray(filtered); }; - public bulkFetchUsage = async (callCluster: LegacyAPICaller) => { + public bulkFetchUsage = async (callCluster: LegacyAPICaller, esClient: ElasticsearchClient) => { const usageCollectors = this.getFilteredCollectorSet((c) => c instanceof UsageCollector); - return await this.bulkFetch(callCluster, usageCollectors.collectors); + return await this.bulkFetch(callCluster, esClient, usageCollectors.collectors); }; // convert an array of fetched stats results into key/object From 149e51c6ffb828dec16f89281fec76b54381d9c5 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Tue, 1 Sep 2020 16:17:39 -0700 Subject: [PATCH 07/40] Adds license getter for new client --- .../telemetry_collection/get_cluster_info.ts | 3 +- .../telemetry_collection/get_local_license.ts | 62 ++++++++++++++++++- .../telemetry_collection/get_local_stats.ts | 6 +- .../register_collection.ts | 2 +- .../server/collector/collector_set.ts | 6 +- 5 files changed, 67 insertions(+), 12 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 8508bdcb6d4ba2..975b45c17047ff 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -35,7 +35,6 @@ export interface ESClusterInfo { minimum_index_compatibility_version: string; }; } - /** * Get the cluster info from the connected cluster. * @@ -48,5 +47,5 @@ export interface ESClusterInfo { */ export function getClusterInfo(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { const useLegacy = true; - return useLegacy ? callCluster('info') : esClient.info; + return useLegacy ? callCluster('info') : esClient.info(); } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index d41904c6d8e0e8..2f29269ec12a18 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manager/server'; let cachedLicense: ESLicense | undefined; @@ -34,6 +34,52 @@ function fetchLicense(callCluster: LegacyAPICaller, local: boolean) { }); } +/** + * Get the cluster's license from the connected node. + * + * This is the equivalent of GET /_license?local=true&accept_enterprise=true . + * + * Like any X-Pack related API, X-Pack must installed for this to work. + */ +async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClient) { + let response; + try { + // Fetching the license from the local node is cheaper than getting it from the master node and good enough + const { body } = await esClient.license.get<{ license: ESLicense }>({ + local: true, + accept_enterprise: true, + }); + cachedLicense = body.license; + response = body.license; + } catch (err) { + // if there is an error, try to get the license from the master node: + if (cachedLicense) { + try { + const { body } = await esClient.license.get<{ license: ESLicense }>({ + local: false, + accept_enterprise: true, + }); + cachedLicense = body.license; + response = body.license; + } catch (masterError) { + if (masterError.statusCode === 404) { + // the master node doesn't have a license and we assume there isn't a license + cachedLicense = undefined; + response = undefined; + } else { + throw err; + } + } + } + if (err.statusCode === 404) { + cachedLicense = undefined; + } else { + throw err; + } + } + return response; +} + /** * Get the cluster's license from the connected node. * @@ -68,9 +114,19 @@ async function getLicenseFromLocalOrMaster(callCluster: LegacyAPICaller) { return license; } -export const getLocalLicense: LicenseGetter = async (clustersDetails, { callCluster }) => { +export const getLocalLicense: LicenseGetter = async ( + clustersDetails, + { callCluster, esClient } +) => { + const useLegacy = true; + if (!useLegacy) { + const license = await getLicenseFromLocalOrMasterNewClient(esClient); + return clustersDetails.reduce( + (acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), + {} + ); + } const license = await getLicenseFromLocalOrMaster(callCluster); - // It should be called only with 1 cluster element in the clustersDetails array, but doing reduce just in case. return clustersDetails.reduce((acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), {}); }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index 8707b43fa26c96..776f12cb4f8b3c 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -64,9 +64,9 @@ export type TelemetryLocalStats = ReturnType; * Get statistics for all products joined by Elasticsearch cluster. */ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( - clustersDetails, - config, // contains the new esClient already scoped - context + clustersDetails, // array of cluster uuid's + config, // contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end + context // StatsCollectionContext contains logger and version (string) ) => { const { callCluster, usageCollection, esClient } = config; diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts index 5afa0df71c5cc8..34db0507eaa686 100644 --- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts +++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts @@ -47,7 +47,7 @@ export function registerCollection( esCluster: ILegacyClusterClient, esClientGetter: () => IClusterClient | undefined ) { - // use the new ES client internally for clusterDetailsGetter, licenceGetter and, potentially, statsGetter. + // Tina TODO: use the new ES client internally for clusterDetailsGetter, licenceGetter and, potentially, statsGetter. telemetryCollectionManager.setCollection({ esCluster, esClientGetter, diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 77c5036a169c8c..d0337f5d96f17f 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -18,7 +18,7 @@ */ import { snakeCase } from 'lodash'; -import { Logger, LegacyAPICaller, IClusterClient, ElasticsearchClient } from 'kibana/server'; +import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { Collector, CollectorOptions } from './collector'; import { UsageCollector } from './usage_collector'; @@ -118,7 +118,7 @@ export class CollectorSet { }; // all collections eventually pass through bulkFetch. - // teh shape of the reponse is different when using the new ES client as is the error handling. + // the shape of the reponse is different when using the new ES client as is the error handling. public bulkFetch = async ( callCluster: LegacyAPICaller, esClient: ElasticsearchClient, @@ -130,7 +130,7 @@ export class CollectorSet { try { return { type: collector.type, - result: await collector.fetch(callCluster, esClient), // note that the response shape is different between hte legacy and new clients. Each `fetch` method will need to ensure it handles reshaping the data correctly. + result: await collector.fetch(callCluster, esClient), // each collector must ensure they handle the return shape appropriately. }; } catch (err) { this.logger.warn(err); From 12a7af6891e17c0b88fcd3e1b3b434a603945a86 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 2 Sep 2020 16:13:37 -0700 Subject: [PATCH 08/40] Types get_data_telemetry --- .../telemetry_collection/get_cluster_info.ts | 21 +- .../telemetry_collection/get_cluster_stats.ts | 6 +- .../get_data_telemetry/get_data_telemetry.ts | 226 +++++------------- .../get_data_telemetry/index.ts | 9 +- .../get_data_telemetry/types.ts | 118 +++++++++ .../telemetry_collection/get_nodes_usage.ts | 2 +- 6 files changed, 197 insertions(+), 185 deletions(-) create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 975b45c17047ff..34690c56fa5464 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -35,6 +35,17 @@ export interface ESClusterInfo { minimum_index_compatibility_version: string; }; } +export interface ESClusterInfoResponse { + body: ESClusterInfo; + statusCode: number; + headers: object; + warnings: string[]; + meta: object; +} +export async function clusterInfoGetter(esClient: ElasticsearchClient) { + const { body } = ((await esClient.info()) as unknown) as ESClusterInfoResponse; + return body; +} /** * Get the cluster info from the connected cluster. * @@ -44,8 +55,16 @@ export interface ESClusterInfo { * @param {function} esClient The asInternalUser handler (exposed for testing) * * TODO: needs work on using the new client + * The new client always returns an object of the shape, regardless of an error during the request execution: + * { + * body: object | boolean + * statusCode: number + * headers: object + * warnings: [string], + * meta: object + * } */ export function getClusterInfo(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { const useLegacy = true; - return useLegacy ? callCluster('info') : esClient.info(); + return useLegacy ? callCluster('info') : clusterInfoGetter(esClient); } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index ba226c5d63b32f..0240a0a6360ec4 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -26,15 +26,11 @@ import { TIMEOUT } from './constants'; * This is the equivalent to GET /_cluster/stats?timeout=30s. */ export async function getClusterStats(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { - // add some sort of a check to see which client we want or, - // hack: const useLegacy = true; - // alternatively, just replace the call with the new client const legacyClusterStats = await callCluster('cluster.stats', { timeout: TIMEOUT, }); - // new client implementation: - const { body } = await esClient.cluster.stats({ timeout: TIMEOUT }); // the response has changed + const { body } = await esClient.cluster.stats({ timeout: TIMEOUT }); return useLegacy ? legacyClusterStats : body; } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index 97574102432835..0b0f41f34236de 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -16,59 +16,19 @@ * specific language governing permissions and limitations * under the License. */ - +import { ApiResponse, RequestParams } from '@elastic/elasticsearch'; import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from './constants'; import { - DATA_DATASETS_INDEX_PATTERNS_UNIQUE, - DataPatternName, - DataTelemetryType, -} from './constants'; - -export interface DataTelemetryBasePayload { - index_count: number; - ecs_index_count?: number; - doc_count?: number; - size_in_bytes?: number; -} - -export interface DataTelemetryDocument extends DataTelemetryBasePayload { - data_stream?: { - dataset?: string; - type?: DataTelemetryType | string; // The union of types is to help autocompletion with some known `data_stream.type`s - }; - package?: { - name: string; - }; - shipper?: string; - pattern_name?: DataPatternName; -} - -export type DataTelemetryPayload = DataTelemetryDocument[]; - -export interface DataTelemetryIndex { - name: string; - packageName?: string; // Populated by Ingest Manager at `_meta.package.name` - managedBy?: string; // Populated by Ingest Manager at `_meta.managed_by` - dataStreamDataset?: string; // To be obtained from `mappings.data_stream.dataset` if it's a constant keyword - dataStreamType?: string; // To be obtained from `mappings.data_stream.type` if it's a constant keyword - shipper?: string; // To be obtained from `_meta.beat` if it's set - isECS?: boolean; // Optional because it can't be obtained via Monitoring. - - // The fields below are optional because we might not be able to obtain them if the user does not - // have access to the index. - docCount?: number; - sizeInBytes?: number; -} - -type AtLeastOne }> = Partial & U[keyof U]; - -type DataDescriptor = AtLeastOne<{ - packageName: string; - dataStreamDataset: string; - dataStreamType: string; - shipper: string; - patternName: DataPatternName; // When found from the list of the index patterns -}>; + DataDescriptor, + AtLeastOne, + DataTelemetryDocument, + DataTelemetryBasePayload, + DataTelemetryPayload, + IndexMappings, + IndexStats, + DataTelemetryIndex, +} from './types'; function findMatchingDescriptors({ name, @@ -171,106 +131,22 @@ export function buildDataTelemetryPayload(indices: DataTelemetryIndex[]): DataTe return [...acc.values()]; } -interface IndexStats { - indices: { - [indexName: string]: { - total: { - docs: { - count: number; - deleted: number; - }; - store: { - size_in_bytes: number; - }; - }; - }; - }; -} - -interface IndexStatistics { - [indexName: string]: { - total: { - docs: { - count: number; - deleted: number; - }; - store: { - size_in_bytes: number; - }; - }; - }; -} -interface IndexMappings { - [indexName: string]: { - mappings: { - _meta?: { - beat?: string; - - // Ingest Manager provided metadata - package?: { - name?: string; - }; - managed_by?: string; // Typically "ingest-manager" - }; - properties: { - data_stream?: { - properties: { - dataset?: { - type: string; - value?: string; - }; - type?: { - type: string; - value?: string; - }; - }; - }; - ecs?: { - properties: { - version?: { - type: string; - }; - }; - }; - }; - }; - }; -} -interface GetMappingsResponse { - body: IndexMappings; - statusCode: number | null; - headers: Record | null; - warnings: string[] | null; - meta: object; -} -interface IndexStatsResponse { - body: IndexStatistics; - statusCode: number | null; - headers: Record | null; - warnings: string[] | null; - meta: object; -} - export async function getDataTelemetry( callCluster: LegacyAPICaller, esClient: ElasticsearchClient ) { + const useLegacy = false; try { const index = [ ...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern), '*-*-*', // Include data-streams aliases `{type}-{dataset}-{namespace}` ]; - // check which client we need to use for now, evetually replace the legacy client with the new one. Hard-coding use of the legacy client for now - const useLegacy = true; - if (!useLegacy) { - const [getMappingsResponse, indexStatsResponse]: [ - GetMappingsResponse, - IndexStatsResponse - ] = await Promise.all([ + if (useLegacy) { + const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([ // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value - esClient.indices.getMapping({ - index: '*', - filter_path: [ + callCluster('indices.getMapping', { + index: '*', // Request all indices because filter_path already filters out the indices without any of those fields + filterPath: [ // _meta.beat tells the shipper '*.mappings._meta.beat', // _meta.package.name tells the Ingest Manager's package @@ -287,47 +163,53 @@ export async function getDataTelemetry( ], }), // GET /_stats/docs,store?level=indices&filter_path=indices.*.total - esClient.indices.stats({ + callCluster('indices.stats', { index, level: 'indices', metric: ['docs', 'store'], - filter_path: ['indices.*.total'], + filterPath: ['indices.*.total'], }), ]); - const indexMappings: IndexMappings = getMappingsResponse.body; - const indexStats: IndexStats = indexStatsResponse.body; + const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices }); } - const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([ - // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value - callCluster('indices.getMapping', { - index: '*', // Request all indices because filter_path already filters out the indices without any of those fields - filterPath: [ - // _meta.beat tells the shipper - '*.mappings._meta.beat', - // _meta.package.name tells the Ingest Manager's package - '*.mappings._meta.package.name', - // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it - '*.mappings._meta.managed_by', - // Does it have `ecs.version` in the mappings? => It follows the ECS conventions - '*.mappings.properties.ecs.properties.version.type', - - // If `data_stream.type` is a `constant_keyword`, it can be reported as a type - '*.mappings.properties.data_stream.properties.type.value', - // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset - '*.mappings.properties.data_stream.properties.dataset.value', - ], - }), - // GET /_stats/docs,store?level=indices&filter_path=indices.*.total - callCluster('indices.stats', { - index, - level: 'indices', - metric: ['docs', 'store'], - filterPath: ['indices.*.total'], - }), + const indexMappingsParams: RequestParams.IndicesGetMapping = { + index: '*', + filter_path: [ + // _meta.beat tells the shipper + '*.mappings._meta.beat', + // _meta.package.name tells the Ingest Manager's package + '*.mappings._meta.package.name', + // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it + '*.mappings._meta.managed_by', + // Does it have `ecs.version` in the mappings? => It follows the ECS conventions + '*.mappings.properties.ecs.properties.version.type', + + // If `data_stream.type` is a `constant_keyword`, it can be reported as a type + '*.mappings.properties.data_stream.properties.type.value', + // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset + '*.mappings.properties.data_stream.properties.dataset.value', + ], + }; + const indicesStatsParams: RequestParams.IndicesStats = { + index: 'index', + level: 'indices', + metric: ['docs', 'store'], + filter_path: ['indices.*.total'], + }; + const [indexMappingsResponse, indexStatsResponse]: [ + ApiResponse, + ApiResponse + ] = await Promise.all([ + esClient.indices.getMapping(indexMappingsParams), + esClient.indices.stats(indicesStatsParams), ]); + const indexMappings = indexMappingsResponse.body as IndexMappings; + const indexStats = indexStatsResponse.body as IndexStats; + const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices }); + const indices = indexNames.map((name) => { const baseIndexInfo = { name, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts index d056d1c9f299f3..3b78a673e654fa 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts @@ -19,9 +19,6 @@ export { DATA_TELEMETRY_ID } from './constants'; -export { - DataTelemetryIndex, - DataTelemetryPayload, - getDataTelemetry, - buildDataTelemetryPayload, -} from './get_data_telemetry'; +export { getDataTelemetry, buildDataTelemetryPayload } from './get_data_telemetry'; + +export { DataTelemetryIndex, DataTelemetryPayload } from './types'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts new file mode 100644 index 00000000000000..7c55dceb31cd0e --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { DataTelemetryType, DataPatternName } from './constants'; + +export interface DataTelemetryBasePayload { + index_count: number; + ecs_index_count?: number; + doc_count?: number; + size_in_bytes?: number; +} + +export interface DataTelemetryDocument extends DataTelemetryBasePayload { + data_stream?: { + dataset?: string; + type?: DataTelemetryType | string; // The union of types is to help autocompletion with some known `data_stream.type`s + }; + package?: { + name: string; + }; + shipper?: string; + pattern_name?: DataPatternName; +} + +export type DataTelemetryPayload = DataTelemetryDocument[]; + +export interface DataTelemetryIndex { + name: string; + packageName?: string; // Populated by Ingest Manager at `_meta.package.name` + managedBy?: string; // Populated by Ingest Manager at `_meta.managed_by` + dataStreamDataset?: string; // To be obtained from `mappings.data_stream.dataset` if it's a constant keyword + dataStreamType?: string; // To be obtained from `mappings.data_stream.type` if it's a constant keyword + shipper?: string; // To be obtained from `_meta.beat` if it's set + isECS?: boolean; // Optional because it can't be obtained via Monitoring. + + // The fields below are optional because we might not be able to obtain them if the user does not + // have access to the index. + docCount?: number; + sizeInBytes?: number; +} + +export type AtLeastOne }> = Partial & U[keyof U]; + +export type DataDescriptor = AtLeastOne<{ + packageName: string; + dataStreamDataset: string; + dataStreamType: string; + shipper: string; + patternName: DataPatternName; // When found from the list of the index patterns +}>; +export interface IndexStats { + indices: { + [indexName: string]: { + total: { + docs: { + count: number; + deleted: number; + }; + store: { + size_in_bytes: number; + }; + }; + }; + }; +} + +export interface IndexMappings { + [indexName: string]: { + mappings: { + _meta?: { + beat?: string; + + // Ingest Manager provided metadata + package?: { + name?: string; + }; + managed_by?: string; // Typically "ingest-manager" + }; + properties: { + data_stream?: { + properties: { + dataset?: { + type: string; + value?: string; + }; + type?: { + type: string; + value?: string; + }; + }; + }; + ecs?: { + properties: { + version?: { + type: string; + }; + }; + }; + }; + }; + }; +} diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index a977d9fb57b470..c1cbb5ba0d296e 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -58,7 +58,7 @@ export async function fetchNodesUsage( callCluster: LegacyAPICaller, esClient: ElasticsearchClient ): Promise { - const useLegacy = true; + const useLegacy = false; const legacyResponse = await callCluster('transport.request', { method: 'GET', path: '/_nodes/usage', From c3bb50f156423e33341beee2c254c9d82c62a371 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 9 Sep 2020 14:55:33 -0700 Subject: [PATCH 09/40] Review comments WIP --- .../server/telemetry_collection/get_cluster_info.ts | 2 +- .../get_data_telemetry/get_data_telemetry.ts | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 34690c56fa5464..9e9758389cabc6 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -43,7 +43,7 @@ export interface ESClusterInfoResponse { meta: object; } export async function clusterInfoGetter(esClient: ElasticsearchClient) { - const { body } = ((await esClient.info()) as unknown) as ESClusterInfoResponse; + const { body } = await esClient.info(); return body; } /** diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index 0b0f41f34236de..9a9be0ed0f3dad 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -192,22 +192,19 @@ export async function getDataTelemetry( ], }; const indicesStatsParams: RequestParams.IndicesStats = { - index: 'index', + index, level: 'indices', metric: ['docs', 'store'], filter_path: ['indices.*.total'], }; - const [indexMappingsResponse, indexStatsResponse]: [ + const [{ body: indexMappings }, { body: indexStats }]: [ ApiResponse, ApiResponse ] = await Promise.all([ - esClient.indices.getMapping(indexMappingsParams), - esClient.indices.stats(indicesStatsParams), + esClient.indices.getMapping(indexMappingsParams), + esClient.indices.stats(indicesStatsParams), ]); - const indexMappings = indexMappingsResponse.body as IndexMappings; - const indexStats = indexStatsResponse.body as IndexStats; - const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices }); const indices = indexNames.map((name) => { From f9bcee2ca37832d7cde0fea3230cd73cdd521ffb Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 9 Sep 2020 15:23:06 -0700 Subject: [PATCH 10/40] Addresses review comments wrt get_data_telemetry.ts --- .../get_data_telemetry/get_data_telemetry.ts | 114 +++++++++++++++-- .../get_data_telemetry/index.ts | 2 - .../get_data_telemetry/types.ts | 118 ------------------ 3 files changed, 104 insertions(+), 130 deletions(-) delete mode 100644 src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index 9a9be0ed0f3dad..327e3603ea9996 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -18,17 +18,58 @@ */ import { ApiResponse, RequestParams } from '@elastic/elasticsearch'; import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; -import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from './constants'; + import { - DataDescriptor, - AtLeastOne, - DataTelemetryDocument, - DataTelemetryBasePayload, - DataTelemetryPayload, - IndexMappings, - IndexStats, - DataTelemetryIndex, -} from './types'; + DATA_DATASETS_INDEX_PATTERNS_UNIQUE, + DataPatternName, + DataTelemetryType, +} from './constants'; + +export interface DataTelemetryBasePayload { + index_count: number; + ecs_index_count?: number; + doc_count?: number; + size_in_bytes?: number; +} + +export interface DataTelemetryDocument extends DataTelemetryBasePayload { + data_stream?: { + dataset?: string; + type?: DataTelemetryType | string; // The union of types is to help autocompletion with some known `data_stream.type`s + }; + package?: { + name: string; + }; + shipper?: string; + pattern_name?: DataPatternName; +} + +export type DataTelemetryPayload = DataTelemetryDocument[]; + +export interface DataTelemetryIndex { + name: string; + packageName?: string; // Populated by Ingest Manager at `_meta.package.name` + managedBy?: string; // Populated by Ingest Manager at `_meta.managed_by` + dataStreamDataset?: string; // To be obtained from `mappings.data_stream.dataset` if it's a constant keyword + dataStreamType?: string; // To be obtained from `mappings.data_stream.type` if it's a constant keyword + shipper?: string; // To be obtained from `_meta.beat` if it's set + isECS?: boolean; // Optional because it can't be obtained via Monitoring. + + // The fields below are optional because we might not be able to obtain them if the user does not + // have access to the index. + docCount?: number; + sizeInBytes?: number; +} + +type AtLeastOne }> = Partial & U[keyof U]; + +type DataDescriptor = AtLeastOne<{ + packageName: string; + dataStreamDataset: string; + dataStreamType: string; + shipper: string; + patternName: DataPatternName; // When found from the list of the index patterns +}>; function findMatchingDescriptors({ name, @@ -131,6 +172,59 @@ export function buildDataTelemetryPayload(indices: DataTelemetryIndex[]): DataTe return [...acc.values()]; } +interface IndexStats { + indices: { + [indexName: string]: { + total: { + docs: { + count: number; + deleted: number; + }; + store: { + size_in_bytes: number; + }; + }; + }; + }; +} + +interface IndexMappings { + [indexName: string]: { + mappings: { + _meta?: { + beat?: string; + + // Ingest Manager provided metadata + package?: { + name?: string; + }; + managed_by?: string; // Typically "ingest-manager" + }; + properties: { + data_stream?: { + properties: { + dataset?: { + type: string; + value?: string; + }; + type?: { + type: string; + value?: string; + }; + }; + }; + ecs?: { + properties: { + version?: { + type: string; + }; + }; + }; + }; + }; + }; +} + export async function getDataTelemetry( callCluster: LegacyAPICaller, esClient: ElasticsearchClient diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts index 3b78a673e654fa..e10ff0056207d0 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts @@ -20,5 +20,3 @@ export { DATA_TELEMETRY_ID } from './constants'; export { getDataTelemetry, buildDataTelemetryPayload } from './get_data_telemetry'; - -export { DataTelemetryIndex, DataTelemetryPayload } from './types'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts deleted file mode 100644 index 7c55dceb31cd0e..00000000000000 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/types.ts +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { DataTelemetryType, DataPatternName } from './constants'; - -export interface DataTelemetryBasePayload { - index_count: number; - ecs_index_count?: number; - doc_count?: number; - size_in_bytes?: number; -} - -export interface DataTelemetryDocument extends DataTelemetryBasePayload { - data_stream?: { - dataset?: string; - type?: DataTelemetryType | string; // The union of types is to help autocompletion with some known `data_stream.type`s - }; - package?: { - name: string; - }; - shipper?: string; - pattern_name?: DataPatternName; -} - -export type DataTelemetryPayload = DataTelemetryDocument[]; - -export interface DataTelemetryIndex { - name: string; - packageName?: string; // Populated by Ingest Manager at `_meta.package.name` - managedBy?: string; // Populated by Ingest Manager at `_meta.managed_by` - dataStreamDataset?: string; // To be obtained from `mappings.data_stream.dataset` if it's a constant keyword - dataStreamType?: string; // To be obtained from `mappings.data_stream.type` if it's a constant keyword - shipper?: string; // To be obtained from `_meta.beat` if it's set - isECS?: boolean; // Optional because it can't be obtained via Monitoring. - - // The fields below are optional because we might not be able to obtain them if the user does not - // have access to the index. - docCount?: number; - sizeInBytes?: number; -} - -export type AtLeastOne }> = Partial & U[keyof U]; - -export type DataDescriptor = AtLeastOne<{ - packageName: string; - dataStreamDataset: string; - dataStreamType: string; - shipper: string; - patternName: DataPatternName; // When found from the list of the index patterns -}>; -export interface IndexStats { - indices: { - [indexName: string]: { - total: { - docs: { - count: number; - deleted: number; - }; - store: { - size_in_bytes: number; - }; - }; - }; - }; -} - -export interface IndexMappings { - [indexName: string]: { - mappings: { - _meta?: { - beat?: string; - - // Ingest Manager provided metadata - package?: { - name?: string; - }; - managed_by?: string; // Typically "ingest-manager" - }; - properties: { - data_stream?: { - properties: { - dataset?: { - type: string; - value?: string; - }; - type?: { - type: string; - value?: string; - }; - }; - }; - ecs?: { - properties: { - version?: { - type: string; - }; - }; - }; - }; - }; - }; -} From 6fd30885740fd5ef013a3e5655765b96ef3ac576 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 9 Sep 2020 16:00:23 -0700 Subject: [PATCH 11/40] Removes optional undefined from esClientGetter types and adds new client xpack_usage getter --- .../server/plugin.ts | 5 ++-- .../server/types.ts | 4 +-- .../get_stats_with_xpack.ts | 4 +-- .../server/telemetry_collection/get_xpack.ts | 26 +++++++++++++------ 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index d25bb3bb8215fe..821c7651c0e7ac 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -140,10 +140,9 @@ export class TelemetryCollectionManagerPlugin ? collection.esCluster.asScoped(request).callAsCurrentUser : collection.esCluster.callAsInternalUser; // Scope the new elasticsearch Client appropriately and pass to the stats collection config - // Problem: I need a Scopeable request to pass to asScoped. const esClient = config.unencrypted - ? collection.esClientGetter()!.asScoped(config.request).asCurrentUser - : collection.esClientGetter()!.asInternalUser; + ? collection.esClientGetter().asScoped(config.request).asCurrentUser + : collection.esClientGetter().asInternalUser; return { callCluster, start, end, usageCollection, esClient }; } diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index e155ed17b079b6..d6e652b43296e2 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -140,7 +140,7 @@ export interface CollectionConfig< title: string; priority: number; esCluster: ILegacyClusterClient; - esClientGetter: () => IClusterClient | undefined; + esClientGetter: () => IClusterClient; statsGetter: StatsGetter; clusterDetailsGetter: ClusterDetailsGetter; licenseGetter: LicenseGetter; @@ -156,6 +156,6 @@ export interface Collection< licenseGetter: LicenseGetter; clusterDetailsGetter: ClusterDetailsGetter; esCluster: ILegacyClusterClient; - esClientGetter: () => IClusterClient | undefined; + esClientGetter: () => IClusterClient; title: string; } diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts index 3fcd25c31e71e2..165865bc0e92ac 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts @@ -17,9 +17,9 @@ export const getStatsWithXpack: StatsGetter<{}, TelemetryAggregatedStats> = asyn config, context ) { - const { callCluster } = config; + const { callCluster, esClient } = config; const clustersLocalStats = await getLocalStats(clustersDetails, config, context); - const xpack = await getXPackUsage(callCluster).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails. + const xpack = await getXPackUsage(callCluster, esClient).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails. return clustersLocalStats.map((localStats) => { if (xpack) { diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts index 9b69540007e5f7..14fd36bfff8319 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts @@ -5,16 +5,10 @@ */ import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; -/** - * Get the cluster stats from the connected cluster. - * - * This is the equivalent of GET /_xpack/usage?master_timeout=${TIMEOUT} - * - * Like any X-Pack related API, X-Pack must installed for this to work. - */ -export function getXPackUsage(callCluster: CallCluster) { +export function legacyClientXpackUsageGetter(callCluster: CallCluster) { return callCluster('transport.request', { method: 'GET', path: '/_xpack/usage', @@ -23,3 +17,19 @@ export function getXPackUsage(callCluster: CallCluster) { }, }); } + +export async function xpackUsageGetter(esClient: ElasticsearchClient) { + const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); + return body; +} +/** + * Get the cluster stats from the connected cluster. + * + * This is the equivalent of GET /_xpack/usage?master_timeout=${TIMEOUT} + * + * Like any X-Pack related API, X-Pack must installed for this to work. + */ +export function getXPackUsage(callCluster: CallCluster, esClient: ElasticsearchClient) { + const useLegacy = true; + return useLegacy ? legacyClientXpackUsageGetter(callCluster) : xpackUsageGetter(esClient); +} From 4ada48e96b73d2ae6b230d0cd56d917f2d895e8e Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 9 Sep 2020 16:06:54 -0700 Subject: [PATCH 12/40] Temporarily sets the elasticsearchClient as the new ES client in Monitoring Collection's start method --- x-pack/plugins/monitoring/server/plugin.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index 3d9dfd08b9f9be..6321cfd83df63d 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -234,7 +234,13 @@ export class Plugin { }; } - start() {} + start({ elasticsearch }: CoreStart) { + // TODO: For the telemetry plugin to work, we need to provide the new ES client. + // The new client should be inititalized with a similar config to `this.cluster` but, since we're not using + // the new client in Monitoring Telemetry collection yet, setting the local client allos progress for now. + // We will update the client in a follow up PR. + this.elasticsearchClient = elasticsearch.client; + } stop() { if (this.cluster) { From fa3b33fab14b9b8a3069d4166c42351dd1773814 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 14 Sep 2020 11:13:41 -0700 Subject: [PATCH 13/40] WIP review comments --- .../server/plugin.ts | 48 +++++++++++-------- .../server/types.ts | 4 +- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 821c7651c0e7ac..fe2a08e8db0a8c 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -132,6 +132,7 @@ export class TelemetryCollectionManagerPlugin private getStatsCollectionConfig( config: StatsGetterConfig, collection: Collection, + collectionEsClient: IClusterClient, usageCollection: UsageCollectionSetup ): StatsCollectionConfig { const { start, end, request } = config; @@ -141,8 +142,8 @@ export class TelemetryCollectionManagerPlugin : collection.esCluster.callAsInternalUser; // Scope the new elasticsearch Client appropriately and pass to the stats collection config const esClient = config.unencrypted - ? collection.esClientGetter().asScoped(config.request).asCurrentUser - : collection.esClientGetter().asInternalUser; + ? collectionEsClient.asScoped(config.request).asCurrentUser + : collectionEsClient.asInternalUser; return { callCluster, start, end, usageCollection, esClient }; } @@ -152,27 +153,32 @@ export class TelemetryCollectionManagerPlugin return []; } for (const collection of this.collections) { - const statsCollectionConfig = this.getStatsCollectionConfig( - config, - collection, - this.usageCollection - ); - try { - const optInStats = await this.getOptInStatsForCollection( + const collectionEsClient = collection.esClientGetter(); + if (collectionEsClient !== undefined) { + const statsCollectionConfig = this.getStatsCollectionConfig( + config, collection, - optInStatus, - statsCollectionConfig + collectionEsClient, + this.usageCollection ); - if (optInStats && optInStats.length) { - this.logger.debug(`Got Opt In stats using ${collection.title} collection.`); - if (config.unencrypted) { - return optInStats; + + try { + const optInStats = await this.getOptInStatsForCollection( + collection, + optInStatus, + statsCollectionConfig + ); + if (optInStats && optInStats.length) { + this.logger.debug(`Got Opt In stats using ${collection.title} collection.`); + if (config.unencrypted) { + return optInStats; + } + return encryptTelemetry(optInStats, { useProdKey: this.isDistributable }); } - return encryptTelemetry(optInStats, { useProdKey: this.isDistributable }); + } catch (err) { + this.logger.debug(`Failed to collect any opt in stats with registered collections.`); + // swallow error to try next collection; } - } catch (err) { - this.logger.debug(`Failed to collect any opt in stats with registered collections.`); - // swallow error to try next collection; } } @@ -206,10 +212,12 @@ export class TelemetryCollectionManagerPlugin for (const collection of this.collections) { // looping through each of the three collections options we have (grouping of usage collection) - if (collection.esClientGetter() !== undefined) { + const collectionEsClient = collection.esClientGetter(); + if (collectionEsClient !== undefined) { const statsCollectionConfig = this.getStatsCollectionConfig( config, collection, + collectionEsClient, this.usageCollection ); try { diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index d6e652b43296e2..c2c78370a0c068 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -140,7 +140,7 @@ export interface CollectionConfig< title: string; priority: number; esCluster: ILegacyClusterClient; - esClientGetter: () => IClusterClient; + esClientGetter: () => IClusterClient; // --> by now we know that the client getter will return the IClusterClient statsGetter: StatsGetter; clusterDetailsGetter: ClusterDetailsGetter; licenseGetter: LicenseGetter; @@ -156,6 +156,6 @@ export interface Collection< licenseGetter: LicenseGetter; clusterDetailsGetter: ClusterDetailsGetter; esCluster: ILegacyClusterClient; - esClientGetter: () => IClusterClient; + esClientGetter: () => IClusterClient | undefined; // the collection could still return undefined for the es client getter. title: string; } From 354240c9235f2113bc8da47cd1bc318dc4b0da59 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 14 Sep 2020 12:30:56 -0700 Subject: [PATCH 14/40] Fixes type issue --- src/plugins/telemetry/server/plugin.ts | 2 +- .../server/telemetry_collection/register_collection.ts | 2 +- src/plugins/telemetry_collection_manager/server/plugin.ts | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/plugins/telemetry/server/plugin.ts b/src/plugins/telemetry/server/plugin.ts index a0bd3664209811..dfbbe3355e69c9 100644 --- a/src/plugins/telemetry/server/plugin.ts +++ b/src/plugins/telemetry/server/plugin.ts @@ -131,7 +131,7 @@ export class TelemetryPlugin implements Plugin IClusterClient | undefined + esClientGetter: () => IClusterClient ) { // Tina TODO: use the new ES client internally for clusterDetailsGetter, licenceGetter and, potentially, statsGetter. telemetryCollectionManager.setCollection({ diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index fe2a08e8db0a8c..1bc63e9273f95d 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -153,6 +153,7 @@ export class TelemetryCollectionManagerPlugin return []; } for (const collection of this.collections) { + // first fetch the client and make sure it's not undefined. const collectionEsClient = collection.esClientGetter(); if (collectionEsClient !== undefined) { const statsCollectionConfig = this.getStatsCollectionConfig( @@ -207,11 +208,11 @@ export class TelemetryCollectionManagerPlugin if (!this.usageCollection) { return []; } - // This is right before we loop through the collectors to call their fetch methods. - // before looping through each collector and calling its fetch method, we ensure that the esClientGetter returns something, if it doesn't, we skip the collection set (local, xpack_local, monitoring). + // TINA notes: This is right before we loop through the collectors to call their fetch methods. + // TINA notes: before looping through each collector and calling its fetch method, we ensure that the esClientGetter returns something, if it doesn't, we skip the collection set (local, xpack_local, monitoring). for (const collection of this.collections) { - // looping through each of the three collections options we have (grouping of usage collection) + // TINA notes: looping through each of the three collections options we have (grouping of usage collection) to make sure we have the es Client const collectionEsClient = collection.esClientGetter(); if (collectionEsClient !== undefined) { const statsCollectionConfig = this.getStatsCollectionConfig( From 1ebf1e82cf656315b7633e033e4ef3d62dee8c3f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 14 Sep 2020 14:26:09 -0700 Subject: [PATCH 15/40] Adds undefined back to StatsCollectionConfig with comments, sets all clients to use new in boolean --- .../server/telemetry_collection/get_cluster_info.ts | 2 +- .../server/telemetry_collection/get_cluster_stats.ts | 2 +- .../server/telemetry_collection/get_data_telemetry/index.ts | 6 +++++- .../telemetry/server/telemetry_collection/get_kibana.ts | 1 + .../server/telemetry_collection/get_local_license.ts | 2 +- .../server/telemetry_collection/get_local_stats.ts | 3 +++ .../server/telemetry_collection/register_collection.ts | 2 +- src/plugins/telemetry_collection_manager/server/types.ts | 2 +- .../usage_collection/server/collector/collector_set.ts | 5 +++-- .../server/telemetry_collection/get_xpack.ts | 2 +- 10 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 9e9758389cabc6..7676de82f700be 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -65,6 +65,6 @@ export async function clusterInfoGetter(esClient: ElasticsearchClient) { * } */ export function getClusterInfo(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { - const useLegacy = true; + const useLegacy = false; return useLegacy ? callCluster('info') : clusterInfoGetter(esClient); } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index 0240a0a6360ec4..4ef7715244796f 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -26,7 +26,7 @@ import { TIMEOUT } from './constants'; * This is the equivalent to GET /_cluster/stats?timeout=30s. */ export async function getClusterStats(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { - const useLegacy = true; + const useLegacy = false; const legacyClusterStats = await callCluster('cluster.stats', { timeout: TIMEOUT, }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts index e10ff0056207d0..340ee4706cf2ed 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts @@ -19,4 +19,8 @@ export { DATA_TELEMETRY_ID } from './constants'; -export { getDataTelemetry, buildDataTelemetryPayload } from './get_data_telemetry'; +export { + getDataTelemetry, + buildDataTelemetryPayload, + DataTelemetryPayload, +} from './get_data_telemetry'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts index 156aee3a0988f4..2f1d0d3003ed83 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts @@ -81,6 +81,7 @@ export function handleKibanaStats( }; } +// Tina Note: pass both the legacy and new es clients to bulkFetch. BulkFetch calls all the collector fetch methods. export async function getKibana( usageCollection: UsageCollectionSetup, callWithInternalUser: LegacyAPICaller, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index 2f29269ec12a18..b33549b292858e 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -118,7 +118,7 @@ export const getLocalLicense: LicenseGetter = async ( clustersDetails, { callCluster, esClient } ) => { - const useLegacy = true; + const useLegacy = false; if (!useLegacy) { const license = await getLicenseFromLocalOrMasterNewClient(esClient); return clustersDetails.reduce( diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index 776f12cb4f8b3c..e0b65d4183c8fe 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -62,6 +62,9 @@ export type TelemetryLocalStats = ReturnType; /** * Get statistics for all products joined by Elasticsearch cluster. + * @param {Array} cluster uuids + * @param {Object} config contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end + * @param {Object} StatsCollectionContext contains logger and version (string) */ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( clustersDetails, // array of cluster uuid's diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts index ab12be9bbc317e..34db0507eaa686 100644 --- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts +++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts @@ -45,7 +45,7 @@ import { getLocalLicense } from './get_local_license'; export function registerCollection( telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, esCluster: ILegacyClusterClient, - esClientGetter: () => IClusterClient + esClientGetter: () => IClusterClient | undefined ) { // Tina TODO: use the new ES client internally for clusterDetailsGetter, licenceGetter and, potentially, statsGetter. telemetryCollectionManager.setCollection({ diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index c2c78370a0c068..d3e7624a16df2c 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -140,7 +140,7 @@ export interface CollectionConfig< title: string; priority: number; esCluster: ILegacyClusterClient; - esClientGetter: () => IClusterClient; // --> by now we know that the client getter will return the IClusterClient + esClientGetter: () => IClusterClient | undefined; // --> by now we know that the client getter will return the IClusterClient BUT we assure that through a code check statsGetter: StatsGetter; clusterDetailsGetter: ClusterDetailsGetter; licenseGetter: LicenseGetter; diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index d0337f5d96f17f..6861be7f4f76b1 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -118,7 +118,8 @@ export class CollectorSet { }; // all collections eventually pass through bulkFetch. - // the shape of the reponse is different when using the new ES client as is the error handling. + // the shape of the response is different when using the new ES client as is the error handling. + // We'll handle the refactor for using the new client in a follow up PR. public bulkFetch = async ( callCluster: LegacyAPICaller, esClient: ElasticsearchClient, @@ -130,7 +131,7 @@ export class CollectorSet { try { return { type: collector.type, - result: await collector.fetch(callCluster, esClient), // each collector must ensure they handle the return shape appropriately. + result: await collector.fetch(callCluster, esClient), // each collector must ensure they handle the response appropriately. }; } catch (err) { this.logger.warn(err); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts index 14fd36bfff8319..5b7308d18c1d65 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts @@ -30,6 +30,6 @@ export async function xpackUsageGetter(esClient: ElasticsearchClient) { * Like any X-Pack related API, X-Pack must installed for this to work. */ export function getXPackUsage(callCluster: CallCluster, esClient: ElasticsearchClient) { - const useLegacy = true; + const useLegacy = false; return useLegacy ? legacyClientXpackUsageGetter(callCluster) : xpackUsageGetter(esClient); } From 6833a9260023ff9d81c6ff424bbd8589e1c9f34a Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 14 Sep 2020 15:19:31 -0700 Subject: [PATCH 16/40] Removes callCluster and boolean from internal collection --- .../telemetry_collection/get_cluster_info.ts | 26 ++----------- .../telemetry_collection/get_cluster_stats.ts | 14 +++---- .../get_data_telemetry/get_data_telemetry.ts | 37 +------------------ .../telemetry_collection/get_local_stats.ts | 8 ++-- .../telemetry_collection/get_nodes_usage.ts | 19 +++------- 5 files changed, 20 insertions(+), 84 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 7676de82f700be..64a8e57fcb7e60 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; // This can be removed when the ES client improves the types export interface ESClusterInfo { @@ -35,13 +35,7 @@ export interface ESClusterInfo { minimum_index_compatibility_version: string; }; } -export interface ESClusterInfoResponse { - body: ESClusterInfo; - statusCode: number; - headers: object; - warnings: string[]; - meta: object; -} + export async function clusterInfoGetter(esClient: ElasticsearchClient) { const { body } = await esClient.info(); return body; @@ -51,20 +45,8 @@ export async function clusterInfoGetter(esClient: ElasticsearchClient) { * * This is the equivalent to GET / * - * @param {function} callCluster The callWithInternalUser handler (exposed for testing) * @param {function} esClient The asInternalUser handler (exposed for testing) - * - * TODO: needs work on using the new client - * The new client always returns an object of the shape, regardless of an error during the request execution: - * { - * body: object | boolean - * statusCode: number - * headers: object - * warnings: [string], - * meta: object - * } */ -export function getClusterInfo(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { - const useLegacy = false; - return useLegacy ? callCluster('info') : clusterInfoGetter(esClient); +export function getClusterInfo(esClient: ElasticsearchClient) { + return clusterInfoGetter(esClient); } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index 4ef7715244796f..ef80df139e2f2f 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -18,26 +18,22 @@ */ import { ClusterDetailsGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { TIMEOUT } from './constants'; /** * Get the cluster stats from the connected cluster. * * This is the equivalent to GET /_cluster/stats?timeout=30s. */ -export async function getClusterStats(callCluster: LegacyAPICaller, esClient: ElasticsearchClient) { - const useLegacy = false; - const legacyClusterStats = await callCluster('cluster.stats', { - timeout: TIMEOUT, - }); +export async function getClusterStats(esClient: ElasticsearchClient) { const { body } = await esClient.cluster.stats({ timeout: TIMEOUT }); - return useLegacy ? legacyClusterStats : body; + return body; } /** * Get the cluster uuids from the connected cluster. */ -export const getClusterUuids: ClusterDetailsGetter = async ({ callCluster, esClient }) => { - const result = await getClusterStats(callCluster, esClient); +export const getClusterUuids: ClusterDetailsGetter = async ({ esClient }) => { + const result = await getClusterStats(esClient); return [{ clusterUuid: result.cluster_uuid }]; }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index 327e3603ea9996..a9e78618a5a009 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -225,47 +225,12 @@ interface IndexMappings { }; } -export async function getDataTelemetry( - callCluster: LegacyAPICaller, - esClient: ElasticsearchClient -) { - const useLegacy = false; +export async function getDataTelemetry(esClient: ElasticsearchClient) { try { const index = [ ...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern), '*-*-*', // Include data-streams aliases `{type}-{dataset}-{namespace}` ]; - if (useLegacy) { - const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([ - // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value - callCluster('indices.getMapping', { - index: '*', // Request all indices because filter_path already filters out the indices without any of those fields - filterPath: [ - // _meta.beat tells the shipper - '*.mappings._meta.beat', - // _meta.package.name tells the Ingest Manager's package - '*.mappings._meta.package.name', - // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it - '*.mappings._meta.managed_by', - // Does it have `ecs.version` in the mappings? => It follows the ECS conventions - '*.mappings.properties.ecs.properties.version.type', - - // If `data_stream.type` is a `constant_keyword`, it can be reported as a type - '*.mappings.properties.data_stream.properties.type.value', - // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset - '*.mappings.properties.data_stream.properties.dataset.value', - ], - }), - // GET /_stats/docs,store?level=indices&filter_path=indices.*.total - callCluster('indices.stats', { - index, - level: 'indices', - metric: ['docs', 'store'], - filterPath: ['indices.*.total'], - }), - ]); - const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices }); - } const indexMappingsParams: RequestParams.IndicesGetMapping = { index: '*', diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index e0b65d4183c8fe..ba1e1bc9d1b507 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -76,11 +76,11 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( return await Promise.all( clustersDetails.map(async (clustersDetail) => { const [clusterInfo, clusterStats, nodesUsage, kibana, dataTelemetry] = await Promise.all([ - getClusterInfo(callCluster, esClient), // cluster info -> careful here: the new ESClient - getClusterStats(callCluster, esClient), // cluster stats (not to be confused with cluster _state_) - getNodesUsage(callCluster, esClient), // nodes_usage info + getClusterInfo(esClient), // cluster info -> careful here: the new ESClient + getClusterStats(esClient), // cluster stats (not to be confused with cluster _state_) + getNodesUsage(esClient), // nodes_usage info getKibana(usageCollection, callCluster, esClient), - getDataTelemetry(callCluster, esClient), + getDataTelemetry(esClient), ]); return handleLocalStats( clusterInfo, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index c1cbb5ba0d296e..6bd61d79e5ae94 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -44,7 +44,6 @@ export interface NodesFeatureUsageResponse { } export type NodesUsageGetter = ( - callCluster: LegacyAPICaller, esClient: ElasticsearchClient ) => Promise<{ nodes: NodeObj[] | Array<{}> }>; /** @@ -55,19 +54,13 @@ export type NodesUsageGetter = ( * The Nodes usage API was introduced in v6.0.0 */ export async function fetchNodesUsage( - callCluster: LegacyAPICaller, esClient: ElasticsearchClient ): Promise { - const useLegacy = false; - const legacyResponse = await callCluster('transport.request', { - method: 'GET', - path: '/_nodes/usage', - query: { - timeout: TIMEOUT, - }, + const { body } = await esClient.nodes.usage({ + metric: '_all', + timeout: TIMEOUT, }); - const { body } = await esClient.nodes.usage({ metric: '_all', timeout: TIMEOUT }); - return useLegacy ? legacyResponse : body; + return body; } /** @@ -75,8 +68,8 @@ export async function fetchNodesUsage( * @param callCluster APICaller * @returns Object containing array of modified usage information with the node_id nested within the data for that node. */ -export const getNodesUsage: NodesUsageGetter = async (callCluster, esClient) => { - const result = await fetchNodesUsage(callCluster, esClient); +export const getNodesUsage: NodesUsageGetter = async (esClient) => { + const result = await fetchNodesUsage(esClient); const transformedNodes = Object.entries(result?.nodes || {}).map(([key, value]) => ({ ...(value as NodeObj), node_id: key, From 17abcabbef75a34833e83b2c8eab032e73ab09df Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 14 Sep 2020 15:51:52 -0700 Subject: [PATCH 17/40] removes callCluster from xpack_collection --- .../telemetry_collection/get_local_license.ts | 61 +------------------ .../register_collection.ts | 1 - .../get_stats_with_xpack.ts | 4 +- .../server/telemetry_collection/get_xpack.ts | 19 ++---- 4 files changed, 8 insertions(+), 77 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index b33549b292858e..754a711774d144 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -22,18 +22,6 @@ import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manag let cachedLicense: ESLicense | undefined; -function fetchLicense(callCluster: LegacyAPICaller, local: boolean) { - return callCluster<{ license: ESLicense }>('transport.request', { - method: 'GET', - path: '/_license', - query: { - local, - // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. - accept_enterprise: 'true', - }, - }); -} - /** * Get the cluster's license from the connected node. * @@ -80,53 +68,8 @@ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClien return response; } -/** - * Get the cluster's license from the connected node. - * - * This is the equivalent of GET /_license?local=true . - * - * Like any X-Pack related API, X-Pack must installed for this to work. - */ -async function getLicenseFromLocalOrMaster(callCluster: LegacyAPICaller) { - // Fetching the local license is cheaper than getting it from the master and good enough - const { license } = await fetchLicense(callCluster, true).catch(async (err) => { - if (cachedLicense) { - try { - // Fallback to the master node's license info - const response = await fetchLicense(callCluster, false); - return response; - } catch (masterError) { - if (masterError.statusCode === 404) { - // If the master node does not have a license, we can assume there is no license - cachedLicense = undefined; - } else { - // Any other errors from the master node, throw and do not send any telemetry - throw err; - } - } - } - return { license: void 0 }; - }); - - if (license) { - cachedLicense = license; - } - return license; -} - -export const getLocalLicense: LicenseGetter = async ( - clustersDetails, - { callCluster, esClient } -) => { - const useLegacy = false; - if (!useLegacy) { - const license = await getLicenseFromLocalOrMasterNewClient(esClient); - return clustersDetails.reduce( - (acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), - {} - ); - } - const license = await getLicenseFromLocalOrMaster(callCluster); +export const getLocalLicense: LicenseGetter = async (clustersDetails, { esClient }) => { + const license = await getLicenseFromLocalOrMasterNewClient(esClient); // It should be called only with 1 cluster element in the clustersDetails array, but doing reduce just in case. return clustersDetails.reduce((acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), {}); }; diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts index 34db0507eaa686..f3148a014a61aa 100644 --- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts +++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts @@ -47,7 +47,6 @@ export function registerCollection( esCluster: ILegacyClusterClient, esClientGetter: () => IClusterClient | undefined ) { - // Tina TODO: use the new ES client internally for clusterDetailsGetter, licenceGetter and, potentially, statsGetter. telemetryCollectionManager.setCollection({ esCluster, esClientGetter, diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts index 165865bc0e92ac..87e3d0a9613da1 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts @@ -17,9 +17,9 @@ export const getStatsWithXpack: StatsGetter<{}, TelemetryAggregatedStats> = asyn config, context ) { - const { callCluster, esClient } = config; + const { esClient } = config; const clustersLocalStats = await getLocalStats(clustersDetails, config, context); - const xpack = await getXPackUsage(callCluster, esClient).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails. + const xpack = await getXPackUsage(esClient).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails. return clustersLocalStats.map((localStats) => { if (xpack) { diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts index 5b7308d18c1d65..c5c263366afc1e 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts @@ -4,22 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; import { ElasticsearchClient } from 'src/core/server'; +import { XpackUsage } from '@elastic/elasticsearch/api/requestParams'; import { TIMEOUT } from './constants'; -export function legacyClientXpackUsageGetter(callCluster: CallCluster) { - return callCluster('transport.request', { - method: 'GET', - path: '/_xpack/usage', - query: { - master_timeout: TIMEOUT, - }, - }); -} - export async function xpackUsageGetter(esClient: ElasticsearchClient) { - const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); + const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); return body; } /** @@ -29,7 +19,6 @@ export async function xpackUsageGetter(esClient: ElasticsearchClient) { * * Like any X-Pack related API, X-Pack must installed for this to work. */ -export function getXPackUsage(callCluster: CallCluster, esClient: ElasticsearchClient) { - const useLegacy = false; - return useLegacy ? legacyClientXpackUsageGetter(callCluster) : xpackUsageGetter(esClient); +export function getXPackUsage(esClient: ElasticsearchClient) { + return xpackUsageGetter(esClient); } From 632ce43fa9bd8dc5bc6b12d9049585481f8ae551 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 16 Sep 2020 08:19:21 -0700 Subject: [PATCH 18/40] unit test refactors --- .../__tests__/get_cluster_info.js | 39 --- .../__tests__/get_cluster_stats.js | 49 --- .../__tests__/get_local_stats.js | 267 ----------------- .../get_cluster_info.test.ts | 55 ++++ .../get_cluster_stats.test.ts | 41 +++ .../telemetry_collection/get_local_license.ts | 2 +- .../get_local_stats.test.ts | 280 ++++++++++++++++++ .../telemetry_collection/get_local_stats.ts | 4 +- .../get_nodes_usage.test.ts | 65 ++-- .../telemetry_collection/get_nodes_usage.ts | 2 +- .../server/types.ts | 2 +- 11 files changed, 415 insertions(+), 391 deletions(-) delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js deleted file mode 100644 index fe83b76cd11583..00000000000000 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; - -import { getClusterInfo } from '../get_cluster_info'; - -export function mockGetClusterInfo(callCluster, clusterInfo, req) { - callCluster.withArgs(req, 'info').returns(clusterInfo); - callCluster.withArgs('info').returns(clusterInfo); -} - -describe('get_cluster_info', () => { - it('uses callCluster to get info API', () => { - const callCluster = sinon.stub(); - const response = Promise.resolve({}); - - mockGetClusterInfo(callCluster, response); - - expect(getClusterInfo(callCluster)).to.be(response); - }); -}); diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js deleted file mode 100644 index d1354608385f64..00000000000000 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; - -import { TIMEOUT } from '../constants'; -import { getClusterStats } from '../get_cluster_stats'; - -export function mockGetClusterStats(callCluster, clusterStats, req) { - callCluster - .withArgs(req, 'cluster.stats', { - timeout: TIMEOUT, - }) - .returns(clusterStats); - - callCluster - .withArgs('cluster.stats', { - timeout: TIMEOUT, - }) - .returns(clusterStats); -} - -describe.skip('get_cluster_stats', () => { - it('uses callCluster to get cluster.stats API', async () => { - const callCluster = sinon.stub(); - const response = Promise.resolve({}); - - mockGetClusterStats(callCluster, response); - - expect(getClusterStats(callCluster)).to.be(response); - }); -}); diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js deleted file mode 100644 index 8541745faea3b6..00000000000000 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; -import { merge, omit } from 'lodash'; - -import { TIMEOUT } from '../constants'; -import { mockGetClusterInfo } from './get_cluster_info'; -import { mockGetClusterStats } from './get_cluster_stats'; - -import { getLocalStats, handleLocalStats } from '../get_local_stats'; - -const mockUsageCollection = (kibanaUsage = {}) => ({ - bulkFetch: () => kibanaUsage, - toObject: (data) => data, -}); - -const getMockServer = (getCluster = sinon.stub()) => ({ - log(tags, message) { - console.log({ tags, message }); - }, - config() { - return { - get(item) { - switch (item) { - case 'pkg.version': - return '8675309-snapshot'; - default: - throw Error(`unexpected config.get('${item}') received.`); - } - }, - }; - }, - plugins: { - elasticsearch: { getCluster }, - }, -}); -function mockGetNodesUsage(callCluster, nodesUsage, req) { - callCluster - .withArgs( - req, - { - method: 'GET', - path: '/_nodes/usage', - query: { - timeout: TIMEOUT, - }, - }, - 'transport.request' - ) - .returns(nodesUsage); -} - -function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) { - mockGetClusterInfo(callCluster, clusterInfo, req); - mockGetClusterStats(callCluster, clusterStats, req); - mockGetNodesUsage(callCluster, nodesUsage, req); -} - -describe('get_local_stats', () => { - const clusterUuid = 'abc123'; - const clusterName = 'my-cool-cluster'; - const version = '2.3.4'; - const clusterInfo = { - cluster_uuid: clusterUuid, - cluster_name: clusterName, - version: { - number: version, - }, - }; - const nodesUsage = [ - { - node_id: 'some_node_id', - timestamp: 1588617023177, - since: 1588616945163, - rest_actions: { - nodes_usage_action: 1, - create_index_action: 1, - document_get_action: 1, - search_action: 19, - nodes_info_action: 36, - }, - aggregations: { - terms: { - bytes: 2, - }, - scripted_metric: { - other: 7, - }, - }, - }, - ]; - const clusterStats = { - _nodes: { failed: 123 }, - cluster_name: 'real-cool', - indices: { totally: 456 }, - nodes: { yup: 'abc' }, - random: 123, - }; - - const kibana = { - kibana: { - great: 'googlymoogly', - versions: [{ version: '8675309', count: 1 }], - }, - kibana_stats: { - os: { - platform: 'rocky', - platformRelease: 'iv', - }, - }, - localization: { - locale: 'en', - labelsCount: 0, - integrities: {}, - }, - sun: { chances: 5 }, - clouds: { chances: 95 }, - rain: { chances: 2 }, - snow: { chances: 0 }, - }; - - const clusterStatsWithNodesUsage = { - ...clusterStats, - nodes: merge(clusterStats.nodes, { usage: nodesUsage }), - }; - const combinedStatsResult = { - collection: 'local', - cluster_uuid: clusterUuid, - cluster_name: clusterName, - version, - cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'), - stack_stats: { - kibana: { - great: 'googlymoogly', - count: 1, - indices: 1, - os: { - platforms: [{ platform: 'rocky', count: 1 }], - platformReleases: [{ platformRelease: 'iv', count: 1 }], - }, - versions: [{ version: '8675309', count: 1 }], - plugins: { - localization: { - locale: 'en', - labelsCount: 0, - integrities: {}, - }, - sun: { chances: 5 }, - clouds: { chances: 95 }, - rain: { chances: 2 }, - snow: { chances: 0 }, - }, - }, - }, - }; - - const context = { - logger: console, - version: '8.0.0', - }; - - describe('handleLocalStats', () => { - it('returns expected object without xpack and kibana data', () => { - const result = handleLocalStats( - clusterInfo, - clusterStatsWithNodesUsage, - void 0, - void 0, - context - ); - expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); - expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); - expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); - expect(result.version).to.be('2.3.4'); - expect(result.collection).to.be('local'); - expect(result.license).to.be(undefined); - expect(result.stack_stats).to.eql({ kibana: undefined, data: undefined }); - }); - - it('returns expected object with xpack', () => { - const result = handleLocalStats( - clusterInfo, - clusterStatsWithNodesUsage, - void 0, - void 0, - context - ); - const { stack_stats: stack, ...cluster } = result; - expect(cluster.collection).to.be(combinedStatsResult.collection); - expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid); - expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name); - expect(stack.kibana).to.be(undefined); // not mocked for this test - expect(stack.data).to.be(undefined); // not mocked for this test - - expect(cluster.version).to.eql(combinedStatsResult.version); - expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats); - expect(cluster.license).to.eql(combinedStatsResult.license); - expect(stack.xpack).to.eql(combinedStatsResult.stack_stats.xpack); - }); - }); - - describe.skip('getLocalStats', () => { - it('returns expected object without xpack data when X-Pack fails to respond', async () => { - const callClusterUsageFailed = sinon.stub(); - const usageCollection = mockUsageCollection(); - mockGetLocalStats( - callClusterUsageFailed, - Promise.resolve(clusterInfo), - Promise.resolve(clusterStats), - Promise.resolve(nodesUsage) - ); - const result = await getLocalStats([], { - server: getMockServer(), - callCluster: callClusterUsageFailed, - usageCollection, - }); - expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); - expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); - expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); - expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes); - expect(result.version).to.be('2.3.4'); - expect(result.collection).to.be('local'); - - // license and xpack usage info come from the same cluster call - expect(result.license).to.be(undefined); - expect(result.stack_stats.xpack).to.be(undefined); - }); - - it('returns expected object with xpack and kibana data', async () => { - const callCluster = sinon.stub(); - const usageCollection = mockUsageCollection(kibana); - mockGetLocalStats( - callCluster, - Promise.resolve(clusterInfo), - Promise.resolve(clusterStats), - Promise.resolve(nodesUsage) - ); - - const result = await getLocalStats([], { - server: getMockServer(callCluster), - usageCollection, - callCluster, - }); - - expect(result.stack_stats.xpack).to.eql(combinedStatsResult.stack_stats.xpack); - expect(result.stack_stats.kibana).to.eql(combinedStatsResult.stack_stats.kibana); - }); - }); -}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts new file mode 100644 index 00000000000000..ccac8b94756abe --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { getClusterInfo } from './get_cluster_info'; + +export function clearMockGetClusterInfo(esClient: DeeplyMockedKeys) { + esClient.info.mockClear(); +} + +export function mockGetClusterInfo(clusterInfo: any): DeeplyMockedKeys { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.info.mockResolvedValue(clusterInfo); + return esClient; +} + +describe('get_cluster_info using the elasticsearch client', () => { + it('uses the esClient to get info API', () => { + const response = Promise.resolve({ + cluster_uuid: '1234', + cluster_name: 'testCluster', + version: { + number: '7.9.2', + build_flavor: 'string', + build_type: 'string', + build_hash: 'string', + build_date: 'string', + build_snapshot: false, + lucene_version: 'string', + minimum_wire_compatibility_version: 'string', + minimum_index_compatibility_version: 'string', + }, + }); + const esClient = mockGetClusterInfo(response); + expect(getClusterInfo(esClient)).toStrictEqual(response); + clearMockGetClusterInfo(esClient); + }); +}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts new file mode 100644 index 00000000000000..1c566963b58e72 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { getClusterStats } from './get_cluster_stats'; + +export function clearMockGetClusterStats(esClient: DeeplyMockedKeys) { + esClient.cluster.stats.mockClear(); +} + +export function mockGetClusterStats(clusterStats: any): DeeplyMockedKeys { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.cluster.stats.mockResolvedValue(clusterStats); + return esClient; +} + +describe('get_cluster_stats', () => { + it('uses the esClient to get the response from the `cluster.stats` API', async () => { + const response = Promise.resolve({ body: { cluster_uuid: '1234' } }); + const esClient = mockGetClusterStats(response); + expect(getClusterStats(esClient)).toStrictEqual(response); + clearMockGetClusterStats(esClient); + }); +}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index 754a711774d144..c91fcba41a5728 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manager/server'; let cachedLicense: ESLicense | undefined; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts new file mode 100644 index 00000000000000..ac2f9912d1cb75 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts @@ -0,0 +1,280 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// import expect from '@kbn/expect'; +// import sinon from 'sinon'; +import { merge, omit } from 'lodash'; + +import { mockGetClusterInfo } from './get_cluster_info.test'; +import { mockGetClusterStats } from './get_cluster_stats.test'; +import { getLocalStats, handleLocalStats } from './get_local_stats'; +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; + +// const mockUsageCollection = (kibanaUsage = {}) => ({ +// bulkFetch: () => kibanaUsage, +// toObject: (data) => data, +// }); + +// const getMockServer = (getCluster = sinon.stub()) => ({ +// log(tags, message) { +// // eslint-disable-next-line no-console +// console.log({ tags, message }); +// }, +// config() { +// return { +// get(item) { +// switch (item) { +// case 'pkg.version': +// return '8675309-snapshot'; +// default: +// throw Error(`unexpected config.get('${item}') received.`); +// } +// }, +// }; +// }, +// plugins: { +// elasticsearch: { getCluster }, +// }, +// }); +function mockGetNodesUsage(nodesUsage: any) { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.nodes.usage.mockResolvedValueOnce(nodesUsage); + return esClient; +} +// function mockGetNodesUsage(callCluster, nodesUsage, req) { +// callCluster +// .withArgs( +// req, +// { +// method: 'GET', +// path: '/_nodes/usage', +// query: { +// timeout: TIMEOUT, +// }, +// }, +// 'transport.request' +// ) +// .returns(nodesUsage); +// } + +function mockGetLocalStats(clusterInfo: any, clusterStats: any, nodesUsage: any) { + mockGetClusterInfo(clusterInfo); + mockGetClusterStats(clusterStats); + mockGetNodesUsage(nodesUsage); +} + +describe('get_local_stats', () => { + const clusterUuid = 'abc123'; + const clusterName = 'my-cool-cluster'; + const version = '2.3.4'; + const clusterInfo = { + cluster_uuid: clusterUuid, + cluster_name: clusterName, + version: { + number: version, + build_flavor: 'string', + build_type: 'string', + build_hash: 'string', + build_date: 'string', + build_snapshot: false, + lucene_version: 'string', + minimum_wire_compatibility_version: 'string', + minimum_index_compatibility_version: 'string', + }, + }; + const nodesUsage = [ + { + node_id: 'some_node_id', + timestamp: 1588617023177, + since: 1588616945163, + rest_actions: { + nodes_usage_action: 1, + create_index_action: 1, + document_get_action: 1, + search_action: 19, + nodes_info_action: 36, + }, + aggregations: { + terms: { + bytes: 2, + }, + scripted_metric: { + other: 7, + }, + }, + }, + ]; + const clusterStats = { + _nodes: { failed: 123 }, + cluster_name: 'real-cool', + indices: { totally: 456 }, + nodes: { yup: 'abc' }, + random: 123, + }; + + const kibana = { + kibana: { + great: 'googlymoogly', + versions: [{ version: '8675309', count: 1 }], + }, + kibana_stats: { + os: { + platform: 'rocky', + platformRelease: 'iv', + }, + }, + localization: { + locale: 'en', + labelsCount: 0, + integrities: {}, + }, + sun: { chances: 5 }, + clouds: { chances: 95 }, + rain: { chances: 2 }, + snow: { chances: 0 }, + }; + + const clusterStatsWithNodesUsage = { + ...clusterStats, + nodes: merge(clusterStats.nodes, { usage: nodesUsage }), + }; + const combinedStatsResult = { + collection: 'local', + cluster_uuid: clusterUuid, + cluster_name: clusterName, + version, + cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'), + stack_stats: { + kibana: { + great: 'googlymoogly', + count: 1, + indices: 1, + os: { + platforms: [{ platform: 'rocky', count: 1 }], + platformReleases: [{ platformRelease: 'iv', count: 1 }], + }, + versions: [{ version: '8675309', count: 1 }], + plugins: { + localization: { + locale: 'en', + labelsCount: 0, + integrities: {}, + }, + sun: { chances: 5 }, + clouds: { chances: 95 }, + rain: { chances: 2 }, + snow: { chances: 0 }, + }, + }, + }, + }; + + const context = { + logger: console, + version: '8.0.0', + }; + + describe('handleLocalStats', () => { + it('returns expected object without xpack and kibana data', () => { + const result = handleLocalStats( + clusterInfo, + clusterStatsWithNodesUsage, + void 0, + void 0, + context + ); + expect(result.cluster_uuid).toStrictEqual(combinedStatsResult.cluster_uuid); + expect(result.cluster_name).toStrictEqual(combinedStatsResult.cluster_name); + expect(result.cluster_stats).toStrictEqual(combinedStatsResult.cluster_stats); + expect(result.version).toEqual('2.3.4'); + expect(result.collection).toEqual('local'); + expect(Object.keys(result)).not.toContain('license'); + expect(result.stack_stats).toEqual({ kibana: undefined, data: undefined }); + }); + + it('returns expected object with xpack', () => { + const result = handleLocalStats( + clusterInfo, + clusterStatsWithNodesUsage, + void 0, + void 0, + context + ); + const { stack_stats: stack, ...cluster } = result; + expect(cluster.collection).toBe(combinedStatsResult.collection); + expect(cluster.cluster_uuid).toBe(combinedStatsResult.cluster_uuid); + expect(cluster.cluster_name).toBe(combinedStatsResult.cluster_name); + expect(stack.kibana).toBe(undefined); // not mocked for this test + expect(stack.data).toBe(undefined); // not mocked for this test + + expect(cluster.version).toEqual(combinedStatsResult.version); + expect(cluster.cluster_stats).toEqual(combinedStatsResult.cluster_stats); + // expect(cluster.license).toEqual(combinedStatsResult.license); + // expect(stack.xpack).toEqual(combinedStatsResult.stack_stats.xpack); + }); + }); + + // describe.skip('getLocalStats', () => { + // it('returns expected object without xpack data when X-Pack fails to respond', async () => { + // const callClusterUsageFailed = sinon.stub(); + // const usageCollection = mockUsageCollection(); + // mockGetLocalStats( + // callClusterUsageFailed, + // Promise.resolve(clusterInfo), + // Promise.resolve(clusterStats), + // Promise.resolve(nodesUsage) + // ); + // const result = await getLocalStats([], { + // server: getMockServer(), + // callCluster: callClusterUsageFailed, + // usageCollection, + // }); + // expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); + // expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); + // expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); + // expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes); + // expect(result.version).to.be('2.3.4'); + // expect(result.collection).to.be('local'); + + // // license and xpack usage info come from the same cluster call + // expect(result.license).to.be(undefined); + // expect(result.stack_stats.xpack).to.be(undefined); + // }); + + // it('returns expected object with xpack and kibana data', async () => { + // const callCluster = sinon.stub(); + // const usageCollection = mockUsageCollection(kibana); + // mockGetLocalStats( + // callCluster, + // Promise.resolve(clusterInfo), + // Promise.resolve(clusterStats), + // Promise.resolve(nodesUsage) + // ); + + // const result = await getLocalStats([], { + // server: getMockServer(callCluster), + // usageCollection, + // callCluster, + // }); + + // expect(result.stack_stats.xpack).to.eql(combinedStatsResult.stack_stats.xpack); + // expect(result.stack_stats.kibana).to.eql(combinedStatsResult.stack_stats.kibana); + // }); + // }); +}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index ba1e1bc9d1b507..f3459f2ba0d21e 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -40,8 +40,8 @@ export function handleLocalStats( // eslint-disable-next-line @typescript-eslint/naming-convention { cluster_name, cluster_uuid, version }: ESClusterInfo, { _nodes, cluster_name: clusterName, ...clusterStats }: any, - kibana: KibanaUsageStats, - dataTelemetry: DataTelemetryPayload, + kibana: KibanaUsageStats | undefined, + dataTelemetry: DataTelemetryPayload | undefined, context: StatsCollectionContext ) { return { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts index 4e4b0e11b79794..9da7d386deb931 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts @@ -18,7 +18,8 @@ */ import { getNodesUsage } from './get_nodes_usage'; -import { TIMEOUT } from './constants'; +import { ElasticsearchClient } from 'kibana/server'; +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; const mockedNodesFetchResponse = { cluster_name: 'test cluster', @@ -44,37 +45,39 @@ const mockedNodesFetchResponse = { }, }, }; + +export function clearMockFetchNodesUsage(esClient: DeeplyMockedKeys) { + esClient.nodes.usage.mockClear(); +} + +export function mockFetchNodesUsage(nodesUsage: any): DeeplyMockedKeys { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.nodes.usage.mockResolvedValueOnce(nodesUsage); + return esClient; +} + describe('get_nodes_usage', () => { - it('calls fetchNodesUsage', async () => { - const callCluster = jest.fn(); - callCluster.mockResolvedValueOnce(mockedNodesFetchResponse); - await getNodesUsage(callCluster); - expect(callCluster).toHaveBeenCalledWith('transport.request', { - path: '/_nodes/usage', - method: 'GET', - query: { - timeout: TIMEOUT, - }, - }); - }); - it('returns a modified array of node usage data', async () => { - const callCluster = jest.fn(); - callCluster.mockResolvedValueOnce(mockedNodesFetchResponse); - const result = await getNodesUsage(callCluster); - expect(result.nodes).toEqual([ - { - aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } }, - node_id: 'some_node_id', - rest_actions: { - create_index_action: 1, - document_get_action: 1, - nodes_info_action: 36, - nodes_usage_action: 1, - search_action: 19, + it('returns a modified array of nodes usage data', async () => { + const response = Promise.resolve({ body: mockedNodesFetchResponse }); + const esClient = mockFetchNodesUsage(response); + const item = await getNodesUsage(esClient); + expect(item).toStrictEqual({ + nodes: [ + { + aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } }, + node_id: 'some_node_id', + rest_actions: { + create_index_action: 1, + document_get_action: 1, + nodes_info_action: 36, + nodes_usage_action: 1, + search_action: 19, + }, + since: 1588616945163, + timestamp: 1588617023177, }, - since: 1588616945163, - timestamp: 1588617023177, - }, - ]); + ], + }); + clearMockFetchNodesUsage(esClient); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index 6bd61d79e5ae94..4b40b5bff69a57 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { TIMEOUT } from './constants'; export interface NodeAggregation { diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index d3e7624a16df2c..db8bf0e6638f65 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -108,7 +108,7 @@ export interface ESLicense { } export interface StatsCollectionContext { - logger: Logger; + logger: Logger | Console; version: string; } From c6d12687fdd3fe509bfd1e7a49dd9b83dee7ee74 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 17 Sep 2020 15:44:44 -0700 Subject: [PATCH 19/40] refactors get_data_telemetry unit tests --- ...emetry_application_usage_collector.test.ts | 15 ++- .../get_cluster_stats.test.ts | 13 ++- .../get_data_telemetry.test.ts | 102 +++++++++++------- .../get_data_telemetry/get_data_telemetry.ts | 2 +- .../get_nodes_usage.test.ts | 20 ++-- 5 files changed, 96 insertions(+), 56 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts index 709736a37d8026..23a77c2d4c288a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts @@ -17,7 +17,11 @@ * under the License. */ -import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks'; +import { + savedObjectsRepositoryMock, + loggingSystemMock, + elasticsearchServiceMock, +} from '../../../../../core/server/mocks'; import { CollectorOptions, createUsageCollectionSetupMock, @@ -50,6 +54,7 @@ describe('telemetry_application_usage', () => { const getUsageCollector = jest.fn(); const registerType = jest.fn(); const callCluster = jest.fn(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; beforeAll(() => registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector) @@ -62,7 +67,7 @@ describe('telemetry_application_usage', () => { test('if no savedObjectClient initialised, return undefined', async () => { expect(collector.isReady()).toBe(false); - expect(await collector.fetch(callCluster)).toBeUndefined(); + expect(await collector.fetch(callCluster, esClient)).toBeUndefined(); jest.runTimersToTime(ROLL_INDICES_START); }); @@ -80,7 +85,7 @@ describe('telemetry_application_usage', () => { jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run expect(collector.isReady()).toBe(true); - expect(await collector.fetch(callCluster)).toStrictEqual({}); + expect(await collector.fetch(callCluster, esClient)).toStrictEqual({}); expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled(); }); @@ -137,7 +142,7 @@ describe('telemetry_application_usage', () => { jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run - expect(await collector.fetch(callCluster)).toStrictEqual({ + expect(await collector.fetch(callCluster, esClient)).toStrictEqual({ appId: { clicks_total: total + 1 + 10, clicks_7_days: total + 1, @@ -197,7 +202,7 @@ describe('telemetry_application_usage', () => { getUsageCollector.mockImplementation(() => savedObjectClient); - expect(await collector.fetch(callCluster)).toStrictEqual({ + expect(await collector.fetch(callCluster, esClient)).toStrictEqual({ appId: { clicks_total: 1, clicks_7_days: 0, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts index 1c566963b58e72..ebddd7ea93fe13 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts @@ -20,6 +20,7 @@ import { ElasticsearchClient } from 'kibana/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getClusterStats } from './get_cluster_stats'; +import { TIMEOUT } from './constants'; export function clearMockGetClusterStats(esClient: DeeplyMockedKeys) { esClient.cluster.stats.mockClear(); @@ -34,8 +35,16 @@ export function mockGetClusterStats(clusterStats: any): DeeplyMockedKeys { it('uses the esClient to get the response from the `cluster.stats` API', async () => { const response = Promise.resolve({ body: { cluster_uuid: '1234' } }); - const esClient = mockGetClusterStats(response); - expect(getClusterStats(esClient)).toStrictEqual(response); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.cluster.stats.mockImplementationOnce( + // @ts-ignore + async (_params = { timeout: TIMEOUT }) => { + return response; + } + ); + const result = getClusterStats(esClient); + expect(esClient.cluster.stats).toHaveBeenCalledWith({ timeout: TIMEOUT }); + expect(result).toStrictEqual(response); clearMockGetClusterStats(esClient); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts index dee718decdc1f7..422ba1ddda5ed8 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts @@ -19,6 +19,7 @@ import { buildDataTelemetryPayload, getDataTelemetry } from './get_data_telemetry'; import { DATA_DATASETS_INDEX_PATTERNS, DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from './constants'; +import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks'; describe('get_data_telemetry', () => { describe('DATA_DATASETS_INDEX_PATTERNS', () => { @@ -195,13 +196,17 @@ describe('get_data_telemetry', () => { describe('getDataTelemetry', () => { test('it returns the base payload (all 0s) because no indices are found', async () => { - const callCluster = mockCallCluster(); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]); + const esClient = mockEsClient(); + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]); + expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1); + expect(esClient.indices.stats).toHaveBeenCalledTimes(1); + + mockEsClientClear(esClient); }); test('can only see the index mappings, but not the stats', async () => { - const callCluster = mockCallCluster(['filebeat-12314']); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + const esClient = mockEsClient(['filebeat-12314']); + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([ { pattern_name: 'filebeat', shipper: 'filebeat', @@ -209,10 +214,13 @@ describe('get_data_telemetry', () => { ecs_index_count: 0, }, ]); + expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1); + expect(esClient.indices.stats).toHaveBeenCalledTimes(1); + mockEsClientClear(esClient); }); test('can see the mappings and the stats', async () => { - const callCluster = mockCallCluster( + const esClient = mockEsClient( ['filebeat-12314'], { isECS: true }, { @@ -221,7 +229,7 @@ describe('get_data_telemetry', () => { }, } ); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([ { pattern_name: 'filebeat', shipper: 'filebeat', @@ -231,10 +239,11 @@ describe('get_data_telemetry', () => { size_in_bytes: 10, }, ]); + mockEsClientClear(esClient); }); test('find an index that does not match any index pattern but has mappings metadata', async () => { - const callCluster = mockCallCluster( + const esClient = mockEsClient( ['cannot_match_anything'], { isECS: true, dataStreamType: 'traces', shipper: 'my-beat' }, { @@ -245,7 +254,7 @@ describe('get_data_telemetry', () => { }, } ); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([ { data_stream: { dataset: undefined, type: 'traces' }, shipper: 'my-beat', @@ -255,48 +264,63 @@ describe('get_data_telemetry', () => { size_in_bytes: 10, }, ]); + mockEsClientClear(esClient); }); test('return empty array when there is an error', async () => { - const callCluster = jest.fn().mockRejectedValue(new Error('Something went terribly wrong')); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // @ts-ignore + esClient.indices.getMapping.mockRejectedValue(new Error('Something went terribly wrong')); + // @ts-ignore + esClient.indices.stats.mockRejectedValue(new Error('Something went terribly wrong')); + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]); + mockEsClientClear(esClient); }); }); }); - -function mockCallCluster( - indicesMappings: string[] = [], +function mockEsClient( + indicesMappings: string[] = [], // an array of `indices` to get mappings from. { isECS = false, dataStreamDataset = '', dataStreamType = '', shipper = '' } = {}, indexStats: any = {} ) { - return jest.fn().mockImplementation(async (method: string, opts: any) => { - if (method === 'indices.getMapping') { - return Object.fromEntries( - indicesMappings.map((index) => [ - index, - { - mappings: { - ...(shipper && { _meta: { beat: shipper } }), - properties: { - ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }), - ...((dataStreamType || dataStreamDataset) && { - data_stream: { - properties: { - ...(dataStreamDataset && { - dataset: { type: 'constant_keyword', value: dataStreamDataset }, - }), - ...(dataStreamType && { - type: { type: 'constant_keyword', value: dataStreamType }, - }), - }, + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // @ts-ignore + esClient.indices.getMapping.mockImplementationOnce(async () => { + const body = Object.fromEntries( + indicesMappings.map((index) => [ + index, + { + mappings: { + ...(shipper && { _meta: { beat: shipper } }), + properties: { + ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }), + ...((dataStreamType || dataStreamDataset) && { + data_stream: { + properties: { + ...(dataStreamDataset && { + dataset: { type: 'constant_keyword', value: dataStreamDataset }, + }), + ...(dataStreamType && { + type: { type: 'constant_keyword', value: dataStreamType }, + }), }, - }), - }, + }, + }), }, }, - ]) - ); - } - return indexStats; + }, + ]) + ); + return { body }; + }); + // @ts-ignore + esClient.indices.stats.mockImplementationOnce(async () => { + return { body: indexStats }; }); + return esClient; +} + +function mockEsClientClear(esClient: any) { + esClient.indices.getMapping.mockClear(); + esClient.indices.stats.mockClear(); } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index a9e78618a5a009..f51648c62cd3f1 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -17,7 +17,7 @@ * under the License. */ import { ApiResponse, RequestParams } from '@elastic/elasticsearch'; -import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts index 9da7d386deb931..54a8f52913a88a 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts @@ -18,7 +18,8 @@ */ import { getNodesUsage } from './get_nodes_usage'; -import { ElasticsearchClient } from 'kibana/server'; +import { TIMEOUT } from './constants'; +// import { ElasticsearchClient } from 'kibana/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; const mockedNodesFetchResponse = { @@ -46,21 +47,22 @@ const mockedNodesFetchResponse = { }, }; -export function clearMockFetchNodesUsage(esClient: DeeplyMockedKeys) { +export function clearMockFetchNodesUsage(esClient: any) { esClient.nodes.usage.mockClear(); } -export function mockFetchNodesUsage(nodesUsage: any): DeeplyMockedKeys { - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.nodes.usage.mockResolvedValueOnce(nodesUsage); - return esClient; -} - describe('get_nodes_usage', () => { it('returns a modified array of nodes usage data', async () => { const response = Promise.resolve({ body: mockedNodesFetchResponse }); - const esClient = mockFetchNodesUsage(response); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.nodes.usage.mockImplementationOnce( + // @ts-ignore + async (_params = { metric: '_all', timeout: TIMEOUT }) => { + return response; + } + ); const item = await getNodesUsage(esClient); + expect(esClient.nodes.usage).toHaveBeenCalledWith({ metric: '_all', timeout: TIMEOUT }); expect(item).toStrictEqual({ nodes: [ { From 298dbda1a77405f00a46e5640a7568b788cc8976 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Fri, 18 Sep 2020 14:08:49 -0700 Subject: [PATCH 20/40] Refactors usage_collection server unit tests --- .../get_local_stats.test.ts | 41 ++++++++----------- .../server/collector/collector.ts | 2 +- .../server/collector/collector_set.test.ts | 13 +++--- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts index ac2f9912d1cb75..ee240dc5540a0c 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts @@ -19,17 +19,17 @@ // import expect from '@kbn/expect'; // import sinon from 'sinon'; -import { merge, omit } from 'lodash'; +import { merge, omit, get } from 'lodash'; import { mockGetClusterInfo } from './get_cluster_info.test'; import { mockGetClusterStats } from './get_cluster_stats.test'; import { getLocalStats, handleLocalStats } from './get_local_stats'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; -// const mockUsageCollection = (kibanaUsage = {}) => ({ -// bulkFetch: () => kibanaUsage, -// toObject: (data) => data, -// }); +const mockUsageCollection = (kibanaUsage = {}) => ({ + bulkFetch: () => kibanaUsage, + toObject: (data: any) => data, +}); // const getMockServer = (getCluster = sinon.stub()) => ({ // log(tags, message) { @@ -53,25 +53,16 @@ import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; // }, // }); function mockGetNodesUsage(nodesUsage: any) { + const response = Promise.resolve({ body: nodesUsage }); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.nodes.usage.mockResolvedValueOnce(nodesUsage); + esClient.nodes.usage.mockImplementationOnce( + // @ts-ignore + async (_params = { metric: '_all', timeout: TIMEOUT }) => { + return response; + } + ); return esClient; } -// function mockGetNodesUsage(callCluster, nodesUsage, req) { -// callCluster -// .withArgs( -// req, -// { -// method: 'GET', -// path: '/_nodes/usage', -// query: { -// timeout: TIMEOUT, -// }, -// }, -// 'transport.request' -// ) -// .returns(nodesUsage); -// } function mockGetLocalStats(clusterInfo: any, clusterStats: any, nodesUsage: any) { mockGetClusterInfo(clusterInfo); @@ -154,6 +145,7 @@ describe('get_local_stats', () => { ...clusterStats, nodes: merge(clusterStats.nodes, { usage: nodesUsage }), }; + const combinedStatsResult = { collection: 'local', cluster_uuid: clusterUuid, @@ -216,6 +208,7 @@ describe('get_local_stats', () => { void 0, context ); + const { stack_stats: stack, ...cluster } = result; expect(cluster.collection).toBe(combinedStatsResult.collection); expect(cluster.cluster_uuid).toBe(combinedStatsResult.cluster_uuid); @@ -225,8 +218,10 @@ describe('get_local_stats', () => { expect(cluster.version).toEqual(combinedStatsResult.version); expect(cluster.cluster_stats).toEqual(combinedStatsResult.cluster_stats); - // expect(cluster.license).toEqual(combinedStatsResult.license); - // expect(stack.xpack).toEqual(combinedStatsResult.stack_stats.xpack); + expect(get(cluster, 'license', '')).toEqual(get(combinedStatsResult, 'license', '')); + expect(get(stack, 'xpack', '')).toEqual( + get(combinedStatsResult, ['stack_stats', 'xpack'], '') + ); }); }); diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 3895769fbb4be9..365e1ce2013370 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -48,7 +48,7 @@ export interface CollectorOptions { type: string; init?: Function; schema?: MakeSchemaFrom>; // Using Required to enforce all optional keys in the object - fetch: (callCluster: LegacyAPICaller, esClient: ElasticsearchClient) => Promise | T; + fetch: (callCluster: LegacyAPICaller, esClient?: ElasticsearchClient) => Promise | T; /* * A hook for allowing the fetched data payload to be organized into a typed * data model for internal bulk upload. See defaultFormatterForBulkUpload for diff --git a/src/plugins/usage_collection/server/collector/collector_set.test.ts b/src/plugins/usage_collection/server/collector/collector_set.test.ts index 545642c5dcfa3b..3f943ad8bf2ffd 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.test.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.test.ts @@ -21,7 +21,7 @@ import { noop } from 'lodash'; import { Collector } from './collector'; import { CollectorSet } from './collector_set'; import { UsageCollector } from './usage_collector'; -import { loggingSystemMock } from '../../../../core/server/mocks'; +import { elasticsearchServiceMock, loggingSystemMock } from '../../../../core/server/mocks'; const logger = loggingSystemMock.createLogger(); @@ -42,6 +42,7 @@ describe('CollectorSet', () => { }); const mockCallCluster = jest.fn().mockResolvedValue({ passTest: 1000 }); + const mockEsClient = elasticsearchServiceMock.createClusterClient().asInternalUser; it('should throw an error if non-Collector type of object is registered', () => { const collectors = new CollectorSet({ logger }); @@ -85,7 +86,7 @@ describe('CollectorSet', () => { }) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(loggerSpies.debug).toHaveBeenCalledTimes(1); expect(loggerSpies.debug).toHaveBeenCalledWith( 'Fetching data from MY_TEST_COLLECTOR collector' @@ -110,7 +111,7 @@ describe('CollectorSet', () => { let result; try { - result = await collectors.bulkFetch(mockCallCluster); + result = await collectors.bulkFetch(mockCallCluster, mockEsClient); } catch (err) { // Do nothing } @@ -128,7 +129,7 @@ describe('CollectorSet', () => { }) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(result).toStrictEqual([ { type: 'MY_TEST_COLLECTOR', @@ -146,7 +147,7 @@ describe('CollectorSet', () => { } as any) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(result).toStrictEqual([ { type: 'MY_TEST_COLLECTOR', @@ -169,7 +170,7 @@ describe('CollectorSet', () => { }) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(result).toStrictEqual([ { type: 'MY_TEST_COLLECTOR', From 9d2a25d2719f1545f2d0f6dd8d48d224de62fd9f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Fri, 18 Sep 2020 14:43:14 -0700 Subject: [PATCH 21/40] defines type for stats route --- .../usage_collection/server/routes/stats.ts | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/plugins/usage_collection/server/routes/stats.ts b/src/plugins/usage_collection/server/routes/stats.ts index 7c64c9f180319b..2bf73684af7e98 100644 --- a/src/plugins/usage_collection/server/routes/stats.ts +++ b/src/plugins/usage_collection/server/routes/stats.ts @@ -24,6 +24,7 @@ import { Observable } from 'rxjs'; import { first } from 'rxjs/operators'; import { + ElasticsearchClient, IRouter, LegacyAPICaller, MetricsServiceSetup, @@ -38,6 +39,22 @@ const STATS_NOT_READY_MESSAGE = i18n.translate('usageCollection.stats.notReadyMe const SNAPSHOT_REGEX = /-snapshot/i; +export interface ESClusterInfo { + cluster_uuid: string; + cluster_name: string; + version: { + number: string; + build_flavor: string; + build_type: string; + build_hash: string; + build_date: string; + build_snapshot?: boolean; + lucene_version: string; + minimum_wire_compatibility_version: string; + minimum_index_compatibility_version: string; + }; +} + export function registerStatsRoute({ router, config, @@ -61,13 +78,21 @@ export function registerStatsRoute({ metrics: MetricsServiceSetup; overallStatus$: Observable; }) { - const getUsage = async (callCluster: LegacyAPICaller): Promise => { - const usage = await collectorSet.bulkFetchUsage(callCluster); + const getUsage = async ( + callCluster: LegacyAPICaller, + esClient: ElasticsearchClient + ): Promise => { + const usage = await collectorSet.bulkFetchUsage(callCluster, esClient); return collectorSet.toObject(usage); }; - const getClusterUuid = async (callCluster: LegacyAPICaller): Promise => { - const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid' }); + // const getClusterUuid = async (callCluster: LegacyAPICaller): Promise => { + // const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid' }); + // return uuid; + // }; + const getClusterUuid = async (esClient: ElasticsearchClient): Promise => { + const result = await esClient.info(); + const { cluster_uuid: uuid } = result.body; return uuid; }; @@ -96,14 +121,15 @@ export function registerStatsRoute({ let extended; if (isExtended) { const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const collectorsReady = await collectorSet.areAllCollectorsReady(); if (shouldGetUsage && !collectorsReady) { return res.customError({ statusCode: 503, body: { message: STATS_NOT_READY_MESSAGE } }); } - const usagePromise = shouldGetUsage ? getUsage(callCluster) : Promise.resolve({}); - const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(callCluster)]); + const usagePromise = shouldGetUsage ? getUsage(callCluster, esClient) : Promise.resolve({}); + const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(esClient)]); let modifiedUsage = usage; if (isLegacy) { From 39738242594c15a02b441aa07864ad1265e92afd Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Fri, 18 Sep 2020 14:44:17 -0700 Subject: [PATCH 22/40] deletes commented out code --- src/plugins/usage_collection/server/routes/stats.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/plugins/usage_collection/server/routes/stats.ts b/src/plugins/usage_collection/server/routes/stats.ts index 2bf73684af7e98..9c3e5aca777984 100644 --- a/src/plugins/usage_collection/server/routes/stats.ts +++ b/src/plugins/usage_collection/server/routes/stats.ts @@ -86,10 +86,6 @@ export function registerStatsRoute({ return collectorSet.toObject(usage); }; - // const getClusterUuid = async (callCluster: LegacyAPICaller): Promise => { - // const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid' }); - // return uuid; - // }; const getClusterUuid = async (esClient: ElasticsearchClient): Promise => { const result = await esClient.info(); const { cluster_uuid: uuid } = result.body; From 0cafa049f5f46ff154bf086e4a5b1eec93d65613 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Fri, 18 Sep 2020 17:36:57 -0700 Subject: [PATCH 23/40] Refactors get_xpack unit test and converts to jest --- .../__tests__/get_xpack.js | 46 -------- .../get_stats_with_xpack.test.ts | 105 ++++++++++-------- .../telemetry_collection/get_xpack.test.ts | 20 ++++ .../server/telemetry_collection/get_xpack.ts | 9 +- 4 files changed, 83 insertions(+), 97 deletions(-) delete mode 100644 x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js create mode 100644 x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js deleted file mode 100644 index eb03701fd195b5..00000000000000 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; - -import { TIMEOUT } from '../constants'; -import { getXPackUsage } from '../get_xpack'; - -function mockGetXPackUsage(callCluster, usage, req) { - callCluster - .withArgs(req, 'transport.request', { - method: 'GET', - path: '/_xpack/usage', - query: { - master_timeout: TIMEOUT, - }, - }) - .returns(usage); - - callCluster - .withArgs('transport.request', { - method: 'GET', - path: '/_xpack/usage', - query: { - master_timeout: TIMEOUT, - }, - }) - .returns(usage); -} - -describe('get_xpack', () => { - describe('getXPackUsage', () => { - it('uses callCluster to get /_xpack/usage API', () => { - const response = Promise.resolve({}); - const callCluster = sinon.stub(); - - mockGetXPackUsage(callCluster, response); - - expect(getXPackUsage(callCluster)).to.be(response); - }); - }); -}); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts index 24382fb89d3373..10363cd5eeb77d 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { coreMock } from '../../../../../src/core/server/mocks'; +import { coreMock, elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getStatsWithXpack } from './get_stats_with_xpack'; const kibana = { @@ -53,34 +53,43 @@ const mockUsageCollection = (kibanaUsage = kibana) => ({ toObject: (data: any) => data, }); +function clearEsClientMock(esClient: any) { + esClient.xpack.usage.mockClear(); + esClient.license.get.mockClear(); + esClient.nodes.usage.mockClear(); + esClient.info.mockClear(); +} + describe('Telemetry Collection: Get Aggregated Stats', () => { test('OSS-like telemetry (no license nor X-Pack telemetry)', async () => { - const callCluster = jest.fn(async (method: string, options: { path?: string }) => { - switch (method) { - case 'transport.request': - if (options.path === '/_license' || options.path === '/_xpack/usage') { - // eslint-disable-next-line no-throw-literal - throw { statusCode: 404 }; - } else if (options.path === '/_nodes/usage') { - return { - cluster_name: 'test cluster', - nodes: nodesUsage, - }; - } - return {}; - case 'info': - return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } }; - default: - return {}; - } - }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // mock for xpack.usage should throw a 404 for this test + esClient.xpack.usage.mockRejectedValue(new Error('Not Found')); + // mock for license should throw a 404 for this test + esClient.license.get.mockRejectedValue(new Error('Not Found')); + // mock for nodes usage should resolve for this test + esClient.nodes.usage.mockResolvedValue( + // @ts-ignore + Promise.resolve({ body: { cluster_name: 'test cluster', nodes: nodesUsage } }) + ); + // mock for info should resolve for this test + esClient.info.mockResolvedValue( + // @ts-ignore + Promise.resolve({ + body: { + cluster_uuid: 'test', + cluster_name: 'test', + version: { number: '8.0.0' }, + }, + }) + ); const usageCollection = mockUsageCollection(); const context = getContext(); const stats = await getStatsWithXpack( [{ clusterUuid: '1234' }], { - callCluster, + esClient, usageCollection, } as any, context @@ -90,39 +99,44 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { timestamp: expect.any(String), }); }); + clearEsClientMock(esClient); }); test('X-Pack telemetry (license + X-Pack)', async () => { - const callCluster = jest.fn(async (method: string, options: { path?: string }) => { - switch (method) { - case 'transport.request': - if (options.path === '/_license') { - return { - license: { type: 'basic' }, - }; - } - if (options.path === '/_xpack/usage') { - return {}; - } - if (options.path === '/_nodes/usage') { - return { - cluster_name: 'test cluster', - nodes: nodesUsage, - }; - } - case 'info': - return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } }; - default: - return {}; - } - }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // mock for license should return a basic license + esClient.license.get.mockResolvedValue( + // @ts-ignore + Promise.resolve({ body: { license: { type: 'basic' } } }) + ); + // mock for xpack usage should return an empty object + esClient.xpack.usage.mockResolvedValue( + // @ts-ignore + Promise.resolve({ body: {} }) + ); + // mock for nodes usage should return the cluster name and nodes usage + esClient.nodes.usage.mockResolvedValue( + // @ts-ignore + Promise.resolve({ body: { cluster_name: 'test cluster', nodes: nodesUsage } }) + ); + esClient.info.mockResolvedValue( + // @ts-ignore + Promise.resolve({ + body: { + cluster_uuid: 'test', + cluster_name: 'test', + version: { number: '8.0.0' }, + }, + }) + ); + // mock for info should return cluster_uuid, cluster_name and version const usageCollection = mockUsageCollection(); const context = getContext(); const stats = await getStatsWithXpack( [{ clusterUuid: '1234' }], { - callCluster, + esClient, usageCollection, } as any, context @@ -132,5 +146,6 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { timestamp: expect.any(String), }); }); + clearEsClientMock(esClient); }); }); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts new file mode 100644 index 00000000000000..4565715785fb6c --- /dev/null +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { getXPackUsage } from './get_xpack'; + +describe('get_xpack', () => { + describe('getXPackUsage', () => { + it('uses esClient to get /_xpack/usage API', async () => { + const response = Promise.resolve({ body: {} }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // @ts-ignore + esClient.xpack.usage.mockResolvedValue(response); + const result = getXPackUsage(esClient); + expect(result).toEqual(response); + }); + }); +}); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts index c5c263366afc1e..237f1041241a75 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts @@ -8,10 +8,6 @@ import { ElasticsearchClient } from 'src/core/server'; import { XpackUsage } from '@elastic/elasticsearch/api/requestParams'; import { TIMEOUT } from './constants'; -export async function xpackUsageGetter(esClient: ElasticsearchClient) { - const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); - return body; -} /** * Get the cluster stats from the connected cluster. * @@ -19,6 +15,7 @@ export async function xpackUsageGetter(esClient: ElasticsearchClient) { * * Like any X-Pack related API, X-Pack must installed for this to work. */ -export function getXPackUsage(esClient: ElasticsearchClient) { - return xpackUsageGetter(esClient); +export async function getXPackUsage(esClient: ElasticsearchClient) { + const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); + return body; } From cf2a172394d67c9e93b0912787ffafaf832a048c Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Sun, 20 Sep 2020 16:29:02 -0700 Subject: [PATCH 24/40] refactors unit tests in telemetry plugin server --- .../telemetry_plugin_collector.ts | 6 +- .../get_cluster_info.test.ts | 14 +- .../telemetry_collection/get_cluster_info.ts | 24 +-- .../telemetry_collection/get_cluster_stats.ts | 5 +- .../get_data_telemetry/index.ts | 1 + .../server/telemetry_collection/get_kibana.ts | 2 - .../get_local_stats.test.ts | 188 ++++++++---------- .../server/plugin.ts | 7 +- .../vega_visualization.test.js.snap | 4 +- x-pack/plugins/monitoring/server/plugin.ts | 2 +- .../get_all_stats.test.ts | 3 + .../telemetry_collection/get_all_stats.ts | 3 +- .../get_cluster_uuids.test.ts | 6 +- 13 files changed, 121 insertions(+), 144 deletions(-) diff --git a/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts b/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts index 0a33fb7a5f8ec5..05836b8448a688 100644 --- a/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts +++ b/src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts @@ -19,11 +19,7 @@ import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { - ISavedObjectsRepository, - SavedObjectsClient, - IClusterClient, -} from '../../../../../core/server'; +import { ISavedObjectsRepository, SavedObjectsClient } from '../../../../../core/server'; import { getTelemetrySavedObject, TelemetrySavedObject } from '../../telemetry_repository'; import { getTelemetryOptIn, getTelemetrySendUsageFrom } from '../../../common/telemetry_config'; import { UsageCollectionSetup } from '../../../../usage_collection/server'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts index ccac8b94756abe..bbbee31e1672a5 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts @@ -38,14 +38,14 @@ describe('get_cluster_info using the elasticsearch client', () => { cluster_name: 'testCluster', version: { number: '7.9.2', - build_flavor: 'string', - build_type: 'string', - build_hash: 'string', - build_date: 'string', + build_flavor: 'default', + build_type: 'docker', + build_hash: 'b5ca9c58fb664ca8bf', + build_date: '2020-07-21T16:40:44.668009Z', build_snapshot: false, - lucene_version: 'string', - minimum_wire_compatibility_version: 'string', - minimum_index_compatibility_version: 'string', + lucene_version: '8.5.1', + minimum_wire_compatibility_version: '6.8.0', + minimum_index_compatibility_version: '6.0.0-beta1', }, }); const esClient = mockGetClusterInfo(response); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 64a8e57fcb7e60..1fa3de5f984eb5 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -25,21 +25,16 @@ export interface ESClusterInfo { cluster_name: string; version: { number: string; - build_flavor: string; - build_type: string; - build_hash: string; - build_date: string; + build_flavor?: string; + build_type?: string; + build_hash?: string; + build_date?: string; build_snapshot?: boolean; - lucene_version: string; - minimum_wire_compatibility_version: string; - minimum_index_compatibility_version: string; + lucene_version?: string; + minimum_wire_compatibility_version?: string; + minimum_index_compatibility_version?: string; }; } - -export async function clusterInfoGetter(esClient: ElasticsearchClient) { - const { body } = await esClient.info(); - return body; -} /** * Get the cluster info from the connected cluster. * @@ -47,6 +42,7 @@ export async function clusterInfoGetter(esClient: ElasticsearchClient) { * * @param {function} esClient The asInternalUser handler (exposed for testing) */ -export function getClusterInfo(esClient: ElasticsearchClient) { - return clusterInfoGetter(esClient); +export async function getClusterInfo(esClient: ElasticsearchClient) { + const { body } = await esClient.info(); + return body; } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index ef80df139e2f2f..858fae048e7e67 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -34,6 +34,7 @@ export async function getClusterStats(esClient: ElasticsearchClient) { * Get the cluster uuids from the connected cluster. */ export const getClusterUuids: ClusterDetailsGetter = async ({ esClient }) => { - const result = await getClusterStats(esClient); - return [{ clusterUuid: result.cluster_uuid }]; + const { body } = await esClient.cluster.stats({ timeout: TIMEOUT }); + + return [{ clusterUuid: body.cluster_uuid }]; }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts index 340ee4706cf2ed..0e2ab98a24cbaa 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts @@ -23,4 +23,5 @@ export { getDataTelemetry, buildDataTelemetryPayload, DataTelemetryPayload, + DataTelemetryIndex, } from './get_data_telemetry'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts index 2f1d0d3003ed83..8910e8a56c4873 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts @@ -48,7 +48,6 @@ export function handleKibanaStats( logger.warn('No Kibana stats returned from usage collectors'); return; } - const { kibana, kibana_stats: kibanaStats, ...plugins } = response; const os = { @@ -81,7 +80,6 @@ export function handleKibanaStats( }; } -// Tina Note: pass both the legacy and new es clients to bulkFetch. BulkFetch calls all the collector fetch methods. export async function getKibana( usageCollection: UsageCollectionSetup, callWithInternalUser: LegacyAPICaller, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts index ee240dc5540a0c..f6380570eeb7ab 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts @@ -17,12 +17,8 @@ * under the License. */ -// import expect from '@kbn/expect'; -// import sinon from 'sinon'; -import { merge, omit, get } from 'lodash'; +import { merge, omit } from 'lodash'; -import { mockGetClusterInfo } from './get_cluster_info.test'; -import { mockGetClusterStats } from './get_cluster_stats.test'; import { getLocalStats, handleLocalStats } from './get_local_stats'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; @@ -30,46 +26,56 @@ const mockUsageCollection = (kibanaUsage = {}) => ({ bulkFetch: () => kibanaUsage, toObject: (data: any) => data, }); - -// const getMockServer = (getCluster = sinon.stub()) => ({ -// log(tags, message) { -// // eslint-disable-next-line no-console -// console.log({ tags, message }); -// }, -// config() { -// return { -// get(item) { -// switch (item) { -// case 'pkg.version': -// return '8675309-snapshot'; -// default: -// throw Error(`unexpected config.get('${item}') received.`); -// } -// }, -// }; -// }, -// plugins: { -// elasticsearch: { getCluster }, -// }, -// }); -function mockGetNodesUsage(nodesUsage: any) { - const response = Promise.resolve({ body: nodesUsage }); +// set up successful call mocks for info, cluster stats, nodes usage and data telemetry +function mockGetLocalStats(clusterInfo: any, clusterStats: any) { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.nodes.usage.mockImplementationOnce( + esClient.info + // @ts-ignore + .mockResolvedValue( + // @ts-ignore + Promise.resolve({ + body: { ...clusterInfo }, + }) + ); + esClient.cluster.stats + // @ts-ignore + .mockResolvedValue(Promise.resolve({ body: { ...clusterStats } })); + esClient.nodes.usage.mockResolvedValue( // @ts-ignore - async (_params = { metric: '_all', timeout: TIMEOUT }) => { - return response; - } + Promise.resolve({ + body: { + cluster_name: 'testCluster', + nodes: { + some_node_id: { + timestamp: 1588617023177, + since: 1588616945163, + rest_actions: { + nodes_usage_action: 1, + create_index_action: 1, + document_get_action: 1, + search_action: 19, + nodes_info_action: 36, + }, + aggregations: { + terms: { + bytes: 2, + }, + scripted_metric: { + other: 7, + }, + }, + }, + }, + }, + }) ); + // @ts-ignore + esClient.indices.getMapping.mockResolvedValue(Promise.resolve({ body: { mappings: {} } })); + // @ts-ignore + esClient.indices.stats.mockResolvedValue(Promise.resolve({ body: { indices: {} } })); return esClient; } -function mockGetLocalStats(clusterInfo: any, clusterStats: any, nodesUsage: any) { - mockGetClusterInfo(clusterInfo); - mockGetClusterStats(clusterStats); - mockGetNodesUsage(nodesUsage); -} - describe('get_local_stats', () => { const clusterUuid = 'abc123'; const clusterName = 'my-cool-cluster'; @@ -77,17 +83,7 @@ describe('get_local_stats', () => { const clusterInfo = { cluster_uuid: clusterUuid, cluster_name: clusterName, - version: { - number: version, - build_flavor: 'string', - build_type: 'string', - build_hash: 'string', - build_date: 'string', - build_snapshot: false, - lucene_version: 'string', - minimum_wire_compatibility_version: 'string', - minimum_index_compatibility_version: 'string', - }, + version: { number: version }, }; const nodesUsage = [ { @@ -143,7 +139,7 @@ describe('get_local_stats', () => { const clusterStatsWithNodesUsage = { ...clusterStats, - nodes: merge(clusterStats.nodes, { usage: nodesUsage }), + nodes: merge(clusterStats.nodes, { usage: { nodes: nodesUsage } }), }; const combinedStatsResult = { @@ -183,7 +179,7 @@ describe('get_local_stats', () => { }; describe('handleLocalStats', () => { - it('returns expected object without xpack and kibana data', () => { + it('returns expected object without xpack or kibana data', () => { const result = handleLocalStats( clusterInfo, clusterStatsWithNodesUsage, @@ -218,58 +214,46 @@ describe('get_local_stats', () => { expect(cluster.version).toEqual(combinedStatsResult.version); expect(cluster.cluster_stats).toEqual(combinedStatsResult.cluster_stats); - expect(get(cluster, 'license', '')).toEqual(get(combinedStatsResult, 'license', '')); - expect(get(stack, 'xpack', '')).toEqual( - get(combinedStatsResult, ['stack_stats', 'xpack'], '') - ); + expect(Object.keys(cluster).indexOf('license')).toBeLessThan(0); + expect(Object.keys(stack).indexOf('xpack')).toBeLessThan(0); }); }); - // describe.skip('getLocalStats', () => { - // it('returns expected object without xpack data when X-Pack fails to respond', async () => { - // const callClusterUsageFailed = sinon.stub(); - // const usageCollection = mockUsageCollection(); - // mockGetLocalStats( - // callClusterUsageFailed, - // Promise.resolve(clusterInfo), - // Promise.resolve(clusterStats), - // Promise.resolve(nodesUsage) - // ); - // const result = await getLocalStats([], { - // server: getMockServer(), - // callCluster: callClusterUsageFailed, - // usageCollection, - // }); - // expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); - // expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); - // expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); - // expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes); - // expect(result.version).to.be('2.3.4'); - // expect(result.collection).to.be('local'); - - // // license and xpack usage info come from the same cluster call - // expect(result.license).to.be(undefined); - // expect(result.stack_stats.xpack).to.be(undefined); - // }); - - // it('returns expected object with xpack and kibana data', async () => { - // const callCluster = sinon.stub(); - // const usageCollection = mockUsageCollection(kibana); - // mockGetLocalStats( - // callCluster, - // Promise.resolve(clusterInfo), - // Promise.resolve(clusterStats), - // Promise.resolve(nodesUsage) - // ); - - // const result = await getLocalStats([], { - // server: getMockServer(callCluster), - // usageCollection, - // callCluster, - // }); + describe('getLocalStats', () => { + it('returns expected object with kibana data', async () => { + const callCluster = jest.fn(); + const usageCollection = mockUsageCollection(kibana); + const esClient = mockGetLocalStats(clusterInfo, clusterStats); + const response = await getLocalStats( + // @ts-ignore + ['abc123'], + // @ts-ignore + { callCluster, usageCollection, esClient }, + context + ); + const result = response[0]; + expect(result.cluster_uuid).toEqual(combinedStatsResult.cluster_uuid); + expect(result.cluster_name).toEqual(combinedStatsResult.cluster_name); + expect(result.cluster_stats).toEqual(combinedStatsResult.cluster_stats); + expect(result.cluster_stats.nodes).toEqual(combinedStatsResult.cluster_stats.nodes); + expect(result.version).toBe('2.3.4'); + expect(result.collection).toBe('local'); + expect(Object.keys(result).indexOf('license')).toBeLessThan(0); + expect(Object.keys(result.stack_stats).indexOf('xpack')).toBeLessThan(0); + }); - // expect(result.stack_stats.xpack).to.eql(combinedStatsResult.stack_stats.xpack); - // expect(result.stack_stats.kibana).to.eql(combinedStatsResult.stack_stats.kibana); - // }); - // }); + it('returns an empty array when no cluster uuid is provided', async () => { + const callCluster = jest.fn(); + const usageCollection = mockUsageCollection(kibana); + const esClient = mockGetLocalStats(clusterInfo, clusterStats); + const response = await getLocalStats( + [], + // @ts-ignore + { callCluster, usageCollection, esClient }, + context + ); + expect(response).toBeDefined(); + expect(response.length).toEqual(0); + }); + }); }); diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 1bc63e9273f95d..f0417df487f157 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -208,9 +208,6 @@ export class TelemetryCollectionManagerPlugin if (!this.usageCollection) { return []; } - // TINA notes: This is right before we loop through the collectors to call their fetch methods. - // TINA notes: before looping through each collector and calling its fetch method, we ensure that the esClientGetter returns something, if it doesn't, we skip the collection set (local, xpack_local, monitoring). - for (const collection of this.collections) { // TINA notes: looping through each of the three collections options we have (grouping of usage collection) to make sure we have the es Client const collectionEsClient = collection.esClientGetter(); @@ -263,8 +260,8 @@ export class TelemetryCollectionManagerPlugin } const [stats, licenses] = await Promise.all([ - collection.statsGetter(clustersDetails, statsCollectionConfig, context), // todo: use new esClient - collection.licenseGetter(clustersDetails, statsCollectionConfig, context), // todo: use new ESClient + collection.statsGetter(clustersDetails, statsCollectionConfig, context), + collection.licenseGetter(clustersDetails, statsCollectionConfig, context), ]); return stats.map((stat) => { diff --git a/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap b/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap index 639559dff30919..3bd990521b2866 100644 --- a/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap +++ b/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap @@ -4,6 +4,6 @@ exports[`VegaVisualizations VegaVisualization - basics should show vega blank re exports[`VegaVisualizations VegaVisualization - basics should show vega graph (may fail in dev env) 1`] = `"
"`; -exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 1`] = `"
"`; +exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 1`] = `"
"`; -exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 2`] = `"
"`; +exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 2`] = `"
"`; diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index 5047e12aa022bd..ace318f4e3c65c 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -74,7 +74,7 @@ export class Plugin { private monitoringCore = {} as MonitoringCore; private legacyShimDependencies = {} as LegacyShimDependencies; private bulkUploader: IBulkUploader = {} as IBulkUploader; - private elasticsearchClient: IClusterClient; + private elasticsearchClient: IClusterClient | undefined; constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts index f0ad6399c6c72d..89f09d349014ff 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts @@ -15,6 +15,7 @@ describe('get_all_stats', () => { const start = 0; const end = 1; const callCluster = sinon.stub(); + const esClient = sinon.stub(); const esClusters = [ { cluster_uuid: 'a' }, @@ -176,6 +177,7 @@ describe('get_all_stats', () => { [{ clusterUuid: 'a' }], { callCluster: callCluster as any, + esClient: esClient as any, usageCollection: {} as any, start, end, @@ -201,6 +203,7 @@ describe('get_all_stats', () => { [], { callCluster: callCluster as any, + esClient: esClient as any, usageCollection: {} as any, start, end, diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts index 726db1706758d9..1170380b26ac8c 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts @@ -21,7 +21,6 @@ type PromiseReturnType any> = ReturnType extend export interface CustomContext { maxBucketSize: number; } - /** * Get statistics for all products joined by Elasticsearch cluster. * Returns the array of clusters joined with the Kibana and Logstash instances. @@ -29,7 +28,7 @@ export interface CustomContext { */ export const getAllStats: StatsGetter = async ( clustersDetails, - { callCluster, start, end }, + { callCluster, start, end, esClient }, { maxBucketSize } ) => { const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts index 519dcc38875f50..b2f3cb6c61526c 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts @@ -5,6 +5,7 @@ */ import sinon from 'sinon'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { getClusterUuids, fetchClusterUuids, @@ -13,6 +14,7 @@ import { describe('get_cluster_uuids', () => { const callCluster = sinon.stub(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const response = { aggregations: { cluster_uuids: { @@ -30,7 +32,7 @@ describe('get_cluster_uuids', () => { it('returns cluster UUIDs', async () => { callCluster.withArgs('search').returns(Promise.resolve(response)); expect( - await getClusterUuids({ callCluster, start, end, usageCollection: {} as any }, { + await getClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, { maxBucketSize: 1, } as any) ).toStrictEqual(expectedUuids); @@ -41,7 +43,7 @@ describe('get_cluster_uuids', () => { it('searches for clusters', async () => { callCluster.returns(Promise.resolve(response)); expect( - await fetchClusterUuids({ callCluster, start, end, usageCollection: {} as any }, { + await fetchClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, { maxBucketSize: 1, } as any) ).toStrictEqual(response); From 2d7cc63b6b5a99cff769fae012d605cdcb67b887 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 21 Sep 2020 10:48:53 -0700 Subject: [PATCH 25/40] Updates documentation, removes comment --- src/plugins/usage_collection/README.md | 6 +++--- .../monitoring/server/kibana_monitoring/bulk_uploader.js | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/plugins/usage_collection/README.md b/src/plugins/usage_collection/README.md index fd8b8c0aa83c7f..d8edc5bb8d18a7 100644 --- a/src/plugins/usage_collection/README.md +++ b/src/plugins/usage_collection/README.md @@ -86,9 +86,9 @@ Some background: - `MY_USAGE_TYPE` can be any string. It usually matches the plugin name. As a safety mechanism, we double check there are no duplicates at the moment of registering the collector. - The `fetch` method needs to support multiple contexts in which it is called. For example, when stats are pulled from a Kibana Metricbeat module, the Beat calls Kibana's stats API to invoke usage collection. -In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest`, where the request headers are expected to have read privilege on the entire `.kibana' index. +In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest` or `esClient` wraps `asCurrentUser`, where the request headers are expected to have read privilege on the entire `.kibana' index. -Note: there will be many cases where you won't need to use the `callCluster` function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS, or use other clients like a custom SavedObjects client. In that case it's up to the plugin to initialize those clients like the example below: +Note: there will be many cases where you won't need to use the `callCluster` (or `esClient`) function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS, or use other clients like a custom SavedObjects client. In that case it's up to the plugin to initialize those clients like the example below: ```ts // server/plugin.ts @@ -302,4 +302,4 @@ These saved objects are automatically consumed by the stats API and surfaced und By storing these metrics and their counts as key-value pairs, we can add more metrics without having to worry about exceeding the 1000-field soft limit in Elasticsearch. -The only caveat is that it makes it harder to consume in Kibana when analysing each entry in the array separately. In the telemetry team we are working to find a solution to this. We are building a new way of reporting telemetry called [Pulse](../../../rfcs/text/0008_pulse.md) that will help on making these UI-Metrics easier to consume. +The only caveat is that it makes it harder to consume in Kibana when analysing each entry in the array separately. In the telemetry team we are working to find a solution to this. diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index 88147acd84fadc..754c84f80a7db1 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -136,7 +136,6 @@ export class BulkUploader { } return; } - // TINA TODO: add the new client call in here. const data = await usageCollection.bulkFetch(this._cluster.callAsInternalUser); const payload = this.toBulkUploadFormat(compact(data), usageCollection); if (payload && payload.length > 0) { From 731cc20d5fbad92979b15db8cfa940de659b8b04 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 21 Sep 2020 10:50:46 -0700 Subject: [PATCH 26/40] removes changes in monitoring bulk_uploader file --- .../plugins/monitoring/server/kibana_monitoring/bulk_uploader.js | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index 754c84f80a7db1..5d8af8d71b7fce 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -136,6 +136,7 @@ export class BulkUploader { } return; } + const data = await usageCollection.bulkFetch(this._cluster.callAsInternalUser); const payload = this.toBulkUploadFormat(compact(data), usageCollection); if (payload && payload.length > 0) { From 3b305b96f7158fa8de2f22119fb47a9ba5b6791f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 21 Sep 2020 11:37:37 -0700 Subject: [PATCH 27/40] self review --- .../get_cluster_stats.test.ts | 2 +- .../get_local_stats.test.ts | 38 ++++++++++++------- .../telemetry_collection/get_local_stats.ts | 2 +- .../server/plugin.ts | 7 ++-- .../server/types.ts | 4 +- .../get_stats_with_xpack.test.ts | 29 +++++++------- .../telemetry_collection/get_xpack.test.ts | 10 ++--- 7 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts index ebddd7ea93fe13..e5ecea02b3d9ef 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts @@ -37,7 +37,7 @@ describe('get_cluster_stats', () => { const response = Promise.resolve({ body: { cluster_uuid: '1234' } }); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; esClient.cluster.stats.mockImplementationOnce( - // @ts-ignore + // @ts-ignore the method only cares about the response body async (_params = { timeout: TIMEOUT }) => { return response; } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts index f6380570eeb7ab..25f77d21e1f71c 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts @@ -30,18 +30,18 @@ const mockUsageCollection = (kibanaUsage = {}) => ({ function mockGetLocalStats(clusterInfo: any, clusterStats: any) { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; esClient.info - // @ts-ignore + // @ts-ignore we only care about the response body .mockResolvedValue( - // @ts-ignore - Promise.resolve({ + // @ts-ignore we only care about the response body + { body: { ...clusterInfo }, - }) + } ); esClient.cluster.stats - // @ts-ignore - .mockResolvedValue(Promise.resolve({ body: { ...clusterStats } })); + // @ts-ignore we only care about the response body + .mockResolvedValue({ body: { ...clusterStats } }); esClient.nodes.usage.mockResolvedValue( - // @ts-ignore + // @ts-ignore we only care about the response body Promise.resolve({ body: { cluster_name: 'testCluster', @@ -69,13 +69,21 @@ function mockGetLocalStats(clusterInfo: any, clusterStats: any) { }, }) ); - // @ts-ignore - esClient.indices.getMapping.mockResolvedValue(Promise.resolve({ body: { mappings: {} } })); - // @ts-ignore - esClient.indices.stats.mockResolvedValue(Promise.resolve({ body: { indices: {} } })); + // @ts-ignore we only care about the response body + esClient.indices.getMapping.mockResolvedValue({ body: { mappings: {} } }); + // @ts-ignore we only care about the response body + esClient.indices.stats.mockResolvedValue({ body: { indices: {} } }); return esClient; } +function mockGetLocalStatsClear(esClient: any) { + esClient.info.mockClear(); + esClient.cluster.stats.mockClear(); + esClient.nodes.usage.mockClear(); + esClient.indices.getMapping.mockClear(); + esClient.indices.stats.mockClear(); +} + describe('get_local_stats', () => { const clusterUuid = 'abc123'; const clusterName = 'my-cool-cluster'; @@ -225,9 +233,9 @@ describe('get_local_stats', () => { const usageCollection = mockUsageCollection(kibana); const esClient = mockGetLocalStats(clusterInfo, clusterStats); const response = await getLocalStats( - // @ts-ignore + // @ts-ignore we only need the uuid string for this test ['abc123'], - // @ts-ignore + // @ts-ignore we only need `bulkFetch` and `toObject` in this test { callCluster, usageCollection, esClient }, context ); @@ -240,6 +248,7 @@ describe('get_local_stats', () => { expect(result.collection).toBe('local'); expect(Object.keys(result).indexOf('license')).toBeLessThan(0); expect(Object.keys(result.stack_stats).indexOf('xpack')).toBeLessThan(0); + mockGetLocalStatsClear(esClient); }); it('returns an empty array when no cluster uuid is provided', async () => { @@ -248,12 +257,13 @@ describe('get_local_stats', () => { const esClient = mockGetLocalStats(clusterInfo, clusterStats); const response = await getLocalStats( [], - // @ts-ignore + // @ts-ignore we only need `bulkFetch` and `toObject` in this test { callCluster, usageCollection, esClient }, context ); expect(response).toBeDefined(); expect(response.length).toEqual(0); + mockGetLocalStatsClear(esClient); }); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index f3459f2ba0d21e..6244c6fac51d39 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -76,7 +76,7 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( return await Promise.all( clustersDetails.map(async (clustersDetail) => { const [clusterInfo, clusterStats, nodesUsage, kibana, dataTelemetry] = await Promise.all([ - getClusterInfo(esClient), // cluster info -> careful here: the new ESClient + getClusterInfo(esClient), // cluster info getClusterStats(esClient), // cluster stats (not to be confused with cluster _state_) getNodesUsage(esClient), // nodes_usage info getKibana(usageCollection, callCluster, esClient), diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index f0417df487f157..39e5516fe7a325 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -209,7 +209,6 @@ export class TelemetryCollectionManagerPlugin return []; } for (const collection of this.collections) { - // TINA notes: looping through each of the three collections options we have (grouping of usage collection) to make sure we have the es Client const collectionEsClient = collection.esClientGetter(); if (collectionEsClient !== undefined) { const statsCollectionConfig = this.getStatsCollectionConfig( @@ -243,15 +242,15 @@ export class TelemetryCollectionManagerPlugin } private async getUsageForCollection( - collection: Collection, // contains the esClientGetter method - statsCollectionConfig: StatsCollectionConfig // contains the already-scoped esClient + collection: Collection, + statsCollectionConfig: StatsCollectionConfig ): Promise { const context: StatsCollectionContext = { logger: this.logger.get(collection.title), version: this.version, ...collection.customContext, }; - // added call to use new esClient + const clustersDetails = await collection.clusterDetailsGetter(statsCollectionConfig, context); if (clustersDetails.length === 0) { diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index db8bf0e6638f65..6d55882201eea9 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -131,8 +131,6 @@ export type LicenseGetter = {}> = ( context: StatsCollectionContext & CustomContext ) => Promise<{ [clusterUuid: string]: ESLicense | undefined }>; -// Note that CollectionConfig is different to Collection!: CollectionConfig ~= Collection & priority. -// This is daft! export interface CollectionConfig< CustomContext extends Record = {}, T extends BasicStatsPayload = BasicStatsPayload @@ -140,7 +138,7 @@ export interface CollectionConfig< title: string; priority: number; esCluster: ILegacyClusterClient; - esClientGetter: () => IClusterClient | undefined; // --> by now we know that the client getter will return the IClusterClient BUT we assure that through a code check + esClientGetter: () => IClusterClient | undefined; // --> by now we know that the client getter will return the IClusterClient but we assure that through a code check statsGetter: StatsGetter; clusterDetailsGetter: ClusterDetailsGetter; licenseGetter: LicenseGetter; diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts index 10363cd5eeb77d..ff08946275c880 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts @@ -69,19 +69,19 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { esClient.license.get.mockRejectedValue(new Error('Not Found')); // mock for nodes usage should resolve for this test esClient.nodes.usage.mockResolvedValue( - // @ts-ignore - Promise.resolve({ body: { cluster_name: 'test cluster', nodes: nodesUsage } }) + // @ts-ignore we only care about the response body + { body: { cluster_name: 'test cluster', nodes: nodesUsage } } ); // mock for info should resolve for this test esClient.info.mockResolvedValue( - // @ts-ignore - Promise.resolve({ + // @ts-ignore we only care about the response body + { body: { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' }, }, - }) + } ); const usageCollection = mockUsageCollection(); const context = getContext(); @@ -106,30 +106,29 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; // mock for license should return a basic license esClient.license.get.mockResolvedValue( - // @ts-ignore - Promise.resolve({ body: { license: { type: 'basic' } } }) + // @ts-ignore we only care about the response body + { body: { license: { type: 'basic' } } } ); // mock for xpack usage should return an empty object esClient.xpack.usage.mockResolvedValue( - // @ts-ignore - Promise.resolve({ body: {} }) + // @ts-ignore we only care about the response body + { body: {} } ); // mock for nodes usage should return the cluster name and nodes usage esClient.nodes.usage.mockResolvedValue( - // @ts-ignore - Promise.resolve({ body: { cluster_name: 'test cluster', nodes: nodesUsage } }) + // @ts-ignore we only care about the response body + { body: { cluster_name: 'test cluster', nodes: nodesUsage } } ); esClient.info.mockResolvedValue( - // @ts-ignore - Promise.resolve({ + // @ts-ignore we only care about the response body + { body: { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' }, }, - }) + } ); - // mock for info should return cluster_uuid, cluster_name and version const usageCollection = mockUsageCollection(); const context = getContext(); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts index 4565715785fb6c..2ff19799fc30b8 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts @@ -9,12 +9,12 @@ import { getXPackUsage } from './get_xpack'; describe('get_xpack', () => { describe('getXPackUsage', () => { it('uses esClient to get /_xpack/usage API', async () => { - const response = Promise.resolve({ body: {} }); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - // @ts-ignore - esClient.xpack.usage.mockResolvedValue(response); - const result = getXPackUsage(esClient); - expect(result).toEqual(response); + // @ts-ignore we only care about the response body + esClient.xpack.usage.mockResolvedValue({ body: {} }); + const result = await getXPackUsage(esClient); + expect(result).toEqual({}); + esClient.xpack.usage.mockClear(); }); }); }); From 682e89531762edc0b8b61baab2ddc85463ec4347 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 21 Sep 2020 14:55:33 -0700 Subject: [PATCH 28/40] removes exclusive test check --- .../server/telemetry_collection/get_cluster_info.test.ts | 2 +- .../server/telemetry_collection/get_cluster_info.ts | 2 +- .../server/telemetry_collection/get_cluster_stats.test.ts | 2 +- .../server/telemetry_collection/get_cluster_stats.ts | 2 +- .../get_data_telemetry/get_data_telemetry.ts | 2 +- .../server/telemetry_collection/get_local_license.ts | 8 +++++--- .../server/telemetry_collection/get_nodes_usage.ts | 2 +- src/plugins/telemetry_collection_manager/server/plugin.ts | 1 - .../usage_collection/server/collector/collector.ts | 3 ++- .../usage_collection/server/collector/collector_set.ts | 3 ++- 10 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts index bbbee31e1672a5..16f7c8b578ced9 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getClusterInfo } from './get_cluster_info'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 1fa3de5f984eb5..407f3325c3a9f5 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; // This can be removed when the ES client improves the types export interface ESClusterInfo { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts index e5ecea02b3d9ef..15d47452ba90f3 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getClusterStats } from './get_cluster_stats'; import { TIMEOUT } from './constants'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index 858fae048e7e67..d2a64e48786799 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -18,7 +18,7 @@ */ import { ClusterDetailsGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; /** * Get the cluster stats from the connected cluster. diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index f51648c62cd3f1..ab38b022e5e0d3 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -17,7 +17,7 @@ * under the License. */ import { ApiResponse, RequestParams } from '@elastic/elasticsearch'; -import { ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index c91fcba41a5728..312ea1b5b38fd6 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manager/server'; let cachedLicense: ESLicense | undefined; @@ -35,7 +35,8 @@ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClien // Fetching the license from the local node is cheaper than getting it from the master node and good enough const { body } = await esClient.license.get<{ license: ESLicense }>({ local: true, - accept_enterprise: true, + // @ts-ignore this should be a boolean + accept_enterprise: 'true', }); cachedLicense = body.license; response = body.license; @@ -45,7 +46,8 @@ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClien try { const { body } = await esClient.license.get<{ license: ESLicense }>({ local: false, - accept_enterprise: true, + // @ts-ignore this should be a boolean + accept_enterprise: 'true', }); cachedLicense = body.license; response = body.license; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index 4b40b5bff69a57..170e69b511d255 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { ElasticsearchClient } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; export interface NodeAggregation { diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 39e5516fe7a325..0c48bee0f86f28 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -144,7 +144,6 @@ export class TelemetryCollectionManagerPlugin const esClient = config.unencrypted ? collectionEsClient.asScoped(config.request).asCurrentUser : collectionEsClient.asInternalUser; - return { callCluster, start, end, usageCollection, esClient }; } diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 365e1ce2013370..89dce678c0ec60 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -17,7 +17,8 @@ * under the License. */ -import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { Logger, LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; export type CollectorFormatForBulkUpload = (result: T) => { type: string; payload: U }; diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 6861be7f4f76b1..4ced609cd284fb 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -18,7 +18,8 @@ */ import { snakeCase } from 'lodash'; -import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { Logger, LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { Collector, CollectorOptions } from './collector'; import { UsageCollector } from './usage_collector'; From 0e5ae52bd87d92480f75ff70c5fe2b6ea3c4cdbe Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 21 Sep 2020 15:07:44 -0700 Subject: [PATCH 29/40] uses relative path to type --- .../telemetry/server/telemetry_collection/get_cluster_info.ts | 2 +- .../telemetry/server/telemetry_collection/get_cluster_stats.ts | 2 +- .../get_data_telemetry/get_data_telemetry.ts | 2 +- .../telemetry/server/telemetry_collection/get_local_license.ts | 2 +- .../telemetry/server/telemetry_collection/get_nodes_usage.ts | 2 +- src/plugins/usage_collection/server/collector/collector.ts | 2 +- src/plugins/usage_collection/server/collector/collector_set.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 407f3325c3a9f5..b90d6ed5df6b8e 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; // This can be removed when the ES client improves the types export interface ESClusterInfo { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index d2a64e48786799..4543fde2ebc91a 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -18,7 +18,7 @@ */ import { ClusterDetailsGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; import { TIMEOUT } from './constants'; /** * Get the cluster stats from the connected cluster. diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index ab38b022e5e0d3..20c5f00c88b485 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -17,7 +17,7 @@ * under the License. */ import { ApiResponse, RequestParams } from '@elastic/elasticsearch'; -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../../src/core/server'; import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index 312ea1b5b38fd6..9ed4e1aa2b289b 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -17,8 +17,8 @@ * under the License. */ -import { ElasticsearchClient } from 'src/core/server'; import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manager/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; let cachedLicense: ESLicense | undefined; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index 170e69b511d255..a59bb7c537db3c 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; import { TIMEOUT } from './constants'; export interface NodeAggregation { diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 89dce678c0ec60..0a87c36a56ca65 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -18,7 +18,7 @@ */ import { Logger, LegacyAPICaller } from 'kibana/server'; -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; export type CollectorFormatForBulkUpload = (result: T) => { type: string; payload: U }; diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 4ced609cd284fb..167d5eaddf5f17 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -19,7 +19,7 @@ import { snakeCase } from 'lodash'; import { Logger, LegacyAPICaller } from 'kibana/server'; -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; import { Collector, CollectorOptions } from './collector'; import { UsageCollector } from './usage_collector'; From 36e44b850da1e44f4cb27cf1d5d4e9f863e1ae4a Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 21 Sep 2020 15:28:42 -0700 Subject: [PATCH 30/40] more type import fixes --- .../server/telemetry_collection/get_cluster_info.test.ts | 2 +- .../server/telemetry_collection/get_cluster_stats.test.ts | 2 +- .../telemetry/server/telemetry_collection/get_kibana.ts | 3 ++- .../server/telemetry_collection/register_collection.ts | 3 ++- src/plugins/telemetry_collection_manager/server/types.ts | 2 +- .../server/telemetry_collection/get_xpack.ts | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts index 16f7c8b578ced9..85bddb4f065a70 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getClusterInfo } from './get_cluster_info'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts index 15d47452ba90f3..216003ed66047a 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from 'src/core/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getClusterStats } from './get_cluster_stats'; import { TIMEOUT } from './constants'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts index 8910e8a56c4873..f8be992f9eac5c 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts @@ -19,8 +19,9 @@ import { omit } from 'lodash'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; +import { LegacyAPICaller } from 'kibana/server'; import { StatsCollectionContext } from 'src/plugins/telemetry_collection_manager/server'; +import { ElasticsearchClient } from '../../../../../src/core/server'; export interface KibanaUsageStats { kibana: { diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts index f3148a014a61aa..9dac4900f5f108 100644 --- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts +++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts @@ -36,8 +36,9 @@ * under the License. */ -import { ILegacyClusterClient, IClusterClient } from 'kibana/server'; +import { ILegacyClusterClient } from 'kibana/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; +import { IClusterClient } from '../../../../../src/core/server'; import { getLocalStats } from './get_local_stats'; import { getClusterUuids } from './get_cluster_stats'; import { getLocalLicense } from './get_local_license'; diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index 6d55882201eea9..44970df30fd16b 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -23,9 +23,9 @@ import { KibanaRequest, ILegacyClusterClient, IClusterClient, - ElasticsearchClient, } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { ElasticsearchClient } from '../../../../src/core/server'; import { TelemetryCollectionManagerPlugin } from './plugin'; export interface TelemetryCollectionManagerPluginSetup { diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts index 237f1041241a75..0952172f7ebe6f 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ElasticsearchClient } from 'src/core/server'; import { XpackUsage } from '@elastic/elasticsearch/api/requestParams'; +import { ElasticsearchClient } from '../../../../../src/core/server'; import { TIMEOUT } from './constants'; /** From d58cd0c1132b62e35680807b1daf6cd59a4f9de7 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Mon, 21 Sep 2020 16:14:58 -0700 Subject: [PATCH 31/40] vega snapshot --- .../public/__snapshots__/vega_visualization.test.js.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap b/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap index 639559dff30919..3bd990521b2866 100644 --- a/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap +++ b/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap @@ -4,6 +4,6 @@ exports[`VegaVisualizations VegaVisualization - basics should show vega blank re exports[`VegaVisualizations VegaVisualization - basics should show vega graph (may fail in dev env) 1`] = `"
"`; -exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 1`] = `"
"`; +exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 1`] = `"
"`; -exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 2`] = `"
"`; +exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 2`] = `"
"`; From 2be0ee705a6983d8bf3bac88954d28323824c744 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Tue, 22 Sep 2020 10:22:23 -0700 Subject: [PATCH 32/40] replaces vega snapshto with one from master --- .../public/__snapshots__/vega_visualization.test.js.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap b/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap index 3bd990521b2866..639559dff30919 100644 --- a/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap +++ b/src/plugins/vis_type_vega/public/__snapshots__/vega_visualization.test.js.snap @@ -4,6 +4,6 @@ exports[`VegaVisualizations VegaVisualization - basics should show vega blank re exports[`VegaVisualizations VegaVisualization - basics should show vega graph (may fail in dev env) 1`] = `"
"`; -exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 1`] = `"
"`; +exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 1`] = `"
"`; -exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 2`] = `"
"`; +exports[`VegaVisualizations VegaVisualization - basics should show vegalite graph and update on resize (may fail in dev env) 2`] = `"
"`; From a9974d22bb47b8c562b48c235b818f507b65ea2e Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Tue, 22 Sep 2020 13:37:08 -0700 Subject: [PATCH 33/40] Adds 400 error to catch in get_local_license to guard against calls from oss --- .../server/telemetry_collection/get_local_license.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index 9ed4e1aa2b289b..80f7e073af5d4c 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -28,10 +28,13 @@ let cachedLicense: ESLicense | undefined; * This is the equivalent of GET /_license?local=true&accept_enterprise=true . * * Like any X-Pack related API, X-Pack must installed for this to work. + * + * In OSS we'll get a 400 response using the nwe client and need to catch that error too */ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClient) { let response; try { + // TODO: extract the call into it's own function that accepts the flag for local // Fetching the license from the local node is cheaper than getting it from the master node and good enough const { body } = await esClient.license.get<{ license: ESLicense }>({ local: true, @@ -52,7 +55,7 @@ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClien cachedLicense = body.license; response = body.license; } catch (masterError) { - if (masterError.statusCode === 404) { + if ([400, 404].includes(masterError.statusCode)) { // the master node doesn't have a license and we assume there isn't a license cachedLicense = undefined; response = undefined; @@ -61,7 +64,7 @@ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClien } } } - if (err.statusCode === 404) { + if ([400, 404].includes(err.statusCode)) { cachedLicense = undefined; } else { throw err; From 4b5bcafc49fdc6acebcd210fa4d74a814c649a0e Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Tue, 22 Sep 2020 17:36:20 -0700 Subject: [PATCH 34/40] Addresses some PR comments --- .../get_cluster_info.test.ts | 28 ++++++++------- .../telemetry_collection/get_cluster_info.ts | 2 +- .../get_cluster_stats.test.ts | 8 +---- .../telemetry_collection/get_cluster_stats.ts | 2 +- .../get_data_telemetry.test.ts | 13 ------- .../get_data_telemetry/get_data_telemetry.ts | 22 ++++++------ .../server/telemetry_collection/get_kibana.ts | 2 +- .../telemetry_collection/get_local_license.ts | 12 +++---- .../get_local_stats.test.ts | 34 +++++++------------ .../get_nodes_usage.test.ts | 6 ---- .../telemetry_collection/get_nodes_usage.ts | 2 +- .../server/plugin.ts | 2 +- .../usage_collection/server/routes/stats.ts | 18 +--------- .../get_stats_with_xpack.test.ts | 9 ----- .../telemetry_collection/get_xpack.test.ts | 1 - .../server/telemetry_collection/get_xpack.ts | 5 ++- 16 files changed, 53 insertions(+), 113 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts index 85bddb4f065a70..459b18d252e171 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts @@ -17,23 +17,25 @@ * under the License. */ -import { ElasticsearchClient } from '../../../../../src/core/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getClusterInfo } from './get_cluster_info'; -export function clearMockGetClusterInfo(esClient: DeeplyMockedKeys) { - esClient.info.mockClear(); -} - -export function mockGetClusterInfo(clusterInfo: any): DeeplyMockedKeys { +export function mockGetClusterInfo(clusterInfo: any) { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.info.mockResolvedValue(clusterInfo); + esClient.info + // @ts-ignore we only care about the response body + .mockResolvedValue( + // @ts-ignore we only care about the response body + { + body: { ...clusterInfo }, + } + ); return esClient; } describe('get_cluster_info using the elasticsearch client', () => { - it('uses the esClient to get info API', () => { - const response = Promise.resolve({ + it('uses the esClient to get info API', async () => { + const clusterInfo = { cluster_uuid: '1234', cluster_name: 'testCluster', version: { @@ -47,9 +49,9 @@ describe('get_cluster_info using the elasticsearch client', () => { minimum_wire_compatibility_version: '6.8.0', minimum_index_compatibility_version: '6.0.0-beta1', }, - }); - const esClient = mockGetClusterInfo(response); - expect(getClusterInfo(esClient)).toStrictEqual(response); - clearMockGetClusterInfo(esClient); + }; + const esClient = mockGetClusterInfo(clusterInfo); + + expect(await getClusterInfo(esClient)).toStrictEqual(clusterInfo); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index b90d6ed5df6b8e..407f3325c3a9f5 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; // This can be removed when the ES client improves the types export interface ESClusterInfo { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts index 216003ed66047a..81551c0c4d93db 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts @@ -17,16 +17,11 @@ * under the License. */ -import { ElasticsearchClient } from '../../../../../src/core/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getClusterStats } from './get_cluster_stats'; import { TIMEOUT } from './constants'; -export function clearMockGetClusterStats(esClient: DeeplyMockedKeys) { - esClient.cluster.stats.mockClear(); -} - -export function mockGetClusterStats(clusterStats: any): DeeplyMockedKeys { +export function mockGetClusterStats(clusterStats: any) { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; esClient.cluster.stats.mockResolvedValue(clusterStats); return esClient; @@ -45,6 +40,5 @@ describe('get_cluster_stats', () => { const result = getClusterStats(esClient); expect(esClient.cluster.stats).toHaveBeenCalledWith({ timeout: TIMEOUT }); expect(result).toStrictEqual(response); - clearMockGetClusterStats(esClient); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index 4543fde2ebc91a..d2a64e48786799 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -18,7 +18,7 @@ */ import { ClusterDetailsGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; /** * Get the cluster stats from the connected cluster. diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts index 422ba1ddda5ed8..bb5eb7f6b726da 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts @@ -200,8 +200,6 @@ describe('get_data_telemetry', () => { await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]); expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1); expect(esClient.indices.stats).toHaveBeenCalledTimes(1); - - mockEsClientClear(esClient); }); test('can only see the index mappings, but not the stats', async () => { @@ -216,7 +214,6 @@ describe('get_data_telemetry', () => { ]); expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1); expect(esClient.indices.stats).toHaveBeenCalledTimes(1); - mockEsClientClear(esClient); }); test('can see the mappings and the stats', async () => { @@ -239,7 +236,6 @@ describe('get_data_telemetry', () => { size_in_bytes: 10, }, ]); - mockEsClientClear(esClient); }); test('find an index that does not match any index pattern but has mappings metadata', async () => { @@ -264,17 +260,13 @@ describe('get_data_telemetry', () => { size_in_bytes: 10, }, ]); - mockEsClientClear(esClient); }); test('return empty array when there is an error', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - // @ts-ignore esClient.indices.getMapping.mockRejectedValue(new Error('Something went terribly wrong')); - // @ts-ignore esClient.indices.stats.mockRejectedValue(new Error('Something went terribly wrong')); await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]); - mockEsClientClear(esClient); }); }); }); @@ -319,8 +311,3 @@ function mockEsClient( }); return esClient; } - -function mockEsClientClear(esClient: any) { - esClient.indices.getMapping.mockClear(); - esClient.indices.stats.mockClear(); -} diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index 20c5f00c88b485..67769793cbfdf8 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -16,8 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { ApiResponse, RequestParams } from '@elastic/elasticsearch'; -import { ElasticsearchClient } from '../../../../../../src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE, @@ -231,9 +230,9 @@ export async function getDataTelemetry(esClient: ElasticsearchClient) { ...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern), '*-*-*', // Include data-streams aliases `{type}-{dataset}-{namespace}` ]; - - const indexMappingsParams: RequestParams.IndicesGetMapping = { - index: '*', + const indexMappingsParams: { index: string; filter_path: string[] } = { + // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value + index: '*', // Request all indices because filter_path already filters out the indices without any of those fields filter_path: [ // _meta.beat tells the shipper '*.mappings._meta.beat', @@ -250,16 +249,19 @@ export async function getDataTelemetry(esClient: ElasticsearchClient) { '*.mappings.properties.data_stream.properties.dataset.value', ], }; - const indicesStatsParams: RequestParams.IndicesStats = { + const indicesStatsParams: { + index: string | string[] | undefined; + level: 'cluster' | 'indices' | 'shards' | undefined; + metric: string[]; + filter_path: string[]; + } = { + // GET /_stats/docs,store?level=indices&filter_path=indices.*.total index, level: 'indices', metric: ['docs', 'store'], filter_path: ['indices.*.total'], }; - const [{ body: indexMappings }, { body: indexStats }]: [ - ApiResponse, - ApiResponse - ] = await Promise.all([ + const [{ body: indexMappings }, { body: indexStats }] = await Promise.all([ esClient.indices.getMapping(indexMappingsParams), esClient.indices.stats(indicesStatsParams), ]); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts index f8be992f9eac5c..0ef9815a4eadb1 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts @@ -21,7 +21,7 @@ import { omit } from 'lodash'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { LegacyAPICaller } from 'kibana/server'; import { StatsCollectionContext } from 'src/plugins/telemetry_collection_manager/server'; -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; export interface KibanaUsageStats { kibana: { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index 80f7e073af5d4c..47b7962d4178bc 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -18,7 +18,7 @@ */ import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; let cachedLicense: ESLicense | undefined; @@ -31,15 +31,14 @@ let cachedLicense: ESLicense | undefined; * * In OSS we'll get a 400 response using the nwe client and need to catch that error too */ -async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClient) { +async function getLicenseFromLocalOrMaster(esClient: ElasticsearchClient) { let response; try { // TODO: extract the call into it's own function that accepts the flag for local // Fetching the license from the local node is cheaper than getting it from the master node and good enough const { body } = await esClient.license.get<{ license: ESLicense }>({ local: true, - // @ts-ignore this should be a boolean - accept_enterprise: 'true', + accept_enterprise: true, }); cachedLicense = body.license; response = body.license; @@ -49,8 +48,7 @@ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClien try { const { body } = await esClient.license.get<{ license: ESLicense }>({ local: false, - // @ts-ignore this should be a boolean - accept_enterprise: 'true', + accept_enterprise: true, }); cachedLicense = body.license; response = body.license; @@ -74,7 +72,7 @@ async function getLicenseFromLocalOrMasterNewClient(esClient: ElasticsearchClien } export const getLocalLicense: LicenseGetter = async (clustersDetails, { esClient }) => { - const license = await getLicenseFromLocalOrMasterNewClient(esClient); + const license = await getLicenseFromLocalOrMaster(esClient); // It should be called only with 1 cluster element in the clustersDetails array, but doing reduce just in case. return clustersDetails.reduce((acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), {}); }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts index 25f77d21e1f71c..0c8b0b249f7d11 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts @@ -20,12 +20,15 @@ import { merge, omit } from 'lodash'; import { getLocalStats, handleLocalStats } from './get_local_stats'; +import { usageCollectionPluginMock } from '../../../usage_collection/server/mocks'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; -const mockUsageCollection = (kibanaUsage = {}) => ({ - bulkFetch: () => kibanaUsage, - toObject: (data: any) => data, -}); +function mockUsageCollection(kibanaUsage = {}) { + const usageCollection = usageCollectionPluginMock.createSetupContract(); + usageCollection.bulkFetch = jest.fn().mockResolvedValue(kibanaUsage); + usageCollection.toObject = jest.fn().mockImplementation((data: any) => data); + return usageCollection; +} // set up successful call mocks for info, cluster stats, nodes usage and data telemetry function mockGetLocalStats(clusterInfo: any, clusterStats: any) { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -42,7 +45,7 @@ function mockGetLocalStats(clusterInfo: any, clusterStats: any) { .mockResolvedValue({ body: { ...clusterStats } }); esClient.nodes.usage.mockResolvedValue( // @ts-ignore we only care about the response body - Promise.resolve({ + { body: { cluster_name: 'testCluster', nodes: { @@ -67,7 +70,7 @@ function mockGetLocalStats(clusterInfo: any, clusterStats: any) { }, }, }, - }) + } ); // @ts-ignore we only care about the response body esClient.indices.getMapping.mockResolvedValue({ body: { mappings: {} } }); @@ -76,14 +79,6 @@ function mockGetLocalStats(clusterInfo: any, clusterStats: any) { return esClient; } -function mockGetLocalStatsClear(esClient: any) { - esClient.info.mockClear(); - esClient.cluster.stats.mockClear(); - esClient.nodes.usage.mockClear(); - esClient.indices.getMapping.mockClear(); - esClient.indices.stats.mockClear(); -} - describe('get_local_stats', () => { const clusterUuid = 'abc123'; const clusterName = 'my-cool-cluster'; @@ -233,10 +228,8 @@ describe('get_local_stats', () => { const usageCollection = mockUsageCollection(kibana); const esClient = mockGetLocalStats(clusterInfo, clusterStats); const response = await getLocalStats( - // @ts-ignore we only need the uuid string for this test - ['abc123'], - // @ts-ignore we only need `bulkFetch` and `toObject` in this test - { callCluster, usageCollection, esClient }, + [{ clusterUuid: 'abc123' }], + { callCluster, usageCollection, esClient, start: '', end: '' }, context ); const result = response[0]; @@ -248,7 +241,6 @@ describe('get_local_stats', () => { expect(result.collection).toBe('local'); expect(Object.keys(result).indexOf('license')).toBeLessThan(0); expect(Object.keys(result.stack_stats).indexOf('xpack')).toBeLessThan(0); - mockGetLocalStatsClear(esClient); }); it('returns an empty array when no cluster uuid is provided', async () => { @@ -257,13 +249,11 @@ describe('get_local_stats', () => { const esClient = mockGetLocalStats(clusterInfo, clusterStats); const response = await getLocalStats( [], - // @ts-ignore we only need `bulkFetch` and `toObject` in this test - { callCluster, usageCollection, esClient }, + { callCluster, usageCollection, esClient, start: '', end: '' }, context ); expect(response).toBeDefined(); expect(response.length).toEqual(0); - mockGetLocalStatsClear(esClient); }); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts index 54a8f52913a88a..8cbd1793f6bcca 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts @@ -19,7 +19,6 @@ import { getNodesUsage } from './get_nodes_usage'; import { TIMEOUT } from './constants'; -// import { ElasticsearchClient } from 'kibana/server'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; const mockedNodesFetchResponse = { @@ -47,10 +46,6 @@ const mockedNodesFetchResponse = { }, }; -export function clearMockFetchNodesUsage(esClient: any) { - esClient.nodes.usage.mockClear(); -} - describe('get_nodes_usage', () => { it('returns a modified array of nodes usage data', async () => { const response = Promise.resolve({ body: mockedNodesFetchResponse }); @@ -80,6 +75,5 @@ describe('get_nodes_usage', () => { }, ], }); - clearMockFetchNodesUsage(esClient); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index a59bb7c537db3c..170e69b511d255 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; export interface NodeAggregation { diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 0c48bee0f86f28..e54e7451a670af 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -108,7 +108,7 @@ export class TelemetryCollectionManagerPlugin throw Error('esCluster name must be set for the getCluster method.'); } if (!esClientGetter) { - throw Error('esClientGetter metod not set.'); + throw Error('esClientGetter method not set.'); } if (!clusterDetailsGetter) { throw Error('Cluster UUIds method is not set.'); diff --git a/src/plugins/usage_collection/server/routes/stats.ts b/src/plugins/usage_collection/server/routes/stats.ts index 9c3e5aca777984..1bfe811e87b8fd 100644 --- a/src/plugins/usage_collection/server/routes/stats.ts +++ b/src/plugins/usage_collection/server/routes/stats.ts @@ -39,22 +39,6 @@ const STATS_NOT_READY_MESSAGE = i18n.translate('usageCollection.stats.notReadyMe const SNAPSHOT_REGEX = /-snapshot/i; -export interface ESClusterInfo { - cluster_uuid: string; - cluster_name: string; - version: { - number: string; - build_flavor: string; - build_type: string; - build_hash: string; - build_date: string; - build_snapshot?: boolean; - lucene_version: string; - minimum_wire_compatibility_version: string; - minimum_index_compatibility_version: string; - }; -} - export function registerStatsRoute({ router, config, @@ -87,7 +71,7 @@ export function registerStatsRoute({ }; const getClusterUuid = async (esClient: ElasticsearchClient): Promise => { - const result = await esClient.info(); + const result = await esClient.info<{ cluster_uuid: string }>(); const { cluster_uuid: uuid } = result.body; return uuid; }; diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts index ff08946275c880..a4806cefeef3df 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts @@ -53,13 +53,6 @@ const mockUsageCollection = (kibanaUsage = kibana) => ({ toObject: (data: any) => data, }); -function clearEsClientMock(esClient: any) { - esClient.xpack.usage.mockClear(); - esClient.license.get.mockClear(); - esClient.nodes.usage.mockClear(); - esClient.info.mockClear(); -} - describe('Telemetry Collection: Get Aggregated Stats', () => { test('OSS-like telemetry (no license nor X-Pack telemetry)', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -99,7 +92,6 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { timestamp: expect.any(String), }); }); - clearEsClientMock(esClient); }); test('X-Pack telemetry (license + X-Pack)', async () => { @@ -145,6 +137,5 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { timestamp: expect.any(String), }); }); - clearEsClientMock(esClient); }); }); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts index 2ff19799fc30b8..106df71e46c7e1 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts @@ -14,7 +14,6 @@ describe('get_xpack', () => { esClient.xpack.usage.mockResolvedValue({ body: {} }); const result = await getXPackUsage(esClient); expect(result).toEqual({}); - esClient.xpack.usage.mockClear(); }); }); }); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts index 0952172f7ebe6f..4dbf2052be28cd 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts @@ -4,8 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { XpackUsage } from '@elastic/elasticsearch/api/requestParams'; -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; /** @@ -16,6 +15,6 @@ import { TIMEOUT } from './constants'; * Like any X-Pack related API, X-Pack must installed for this to work. */ export async function getXPackUsage(esClient: ElasticsearchClient) { - const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); + const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); return body; } From 80fa55d80b2ccbf0dcc7c2863092b0bbfabd414f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 23 Sep 2020 09:31:32 -0700 Subject: [PATCH 35/40] Updates data plugin api docs --- src/plugins/data/server/server.api.md | 1 + src/plugins/usage_collection/server/collector/collector.ts | 3 +-- src/plugins/usage_collection/server/collector/collector_set.ts | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 2024e9e7f29743..0495fbb6d36942 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -43,6 +43,7 @@ import { DeleteDocumentParams } from 'elasticsearch'; import { DeleteScriptParams } from 'elasticsearch'; import { DeleteTemplateParams } from 'elasticsearch'; import { Duration } from 'moment'; +import { ElasticsearchClient as ElasticsearchClient_2 } from 'kibana/server'; import { Ensure } from '@kbn/utility-types'; import { EnvironmentMode } from '@kbn/config'; import { ErrorToastOptions } from 'src/core/public/notifications'; diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 0a87c36a56ca65..365e1ce2013370 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -17,8 +17,7 @@ * under the License. */ -import { Logger, LegacyAPICaller } from 'kibana/server'; -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; export type CollectorFormatForBulkUpload = (result: T) => { type: string; payload: U }; diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 167d5eaddf5f17..6861be7f4f76b1 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -18,8 +18,7 @@ */ import { snakeCase } from 'lodash'; -import { Logger, LegacyAPICaller } from 'kibana/server'; -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { Collector, CollectorOptions } from './collector'; import { UsageCollector } from './usage_collector'; From ce3a9a039bde60d2d9f3ed8c99ac44fd057e2ca7 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 23 Sep 2020 16:01:00 -0700 Subject: [PATCH 36/40] Changes elasticsearchClient to telemetryElasticsearchClient in monitoring --- x-pack/plugins/monitoring/server/plugin.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index ace318f4e3c65c..8d3248ddf43697 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -74,7 +74,7 @@ export class Plugin { private monitoringCore = {} as MonitoringCore; private legacyShimDependencies = {} as LegacyShimDependencies; private bulkUploader: IBulkUploader = {} as IBulkUploader; - private elasticsearchClient: IClusterClient | undefined; + private telemetryElasticsearchClient: IClusterClient | undefined; constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; @@ -148,7 +148,7 @@ export class Plugin { registerMonitoringCollection( plugins.telemetryCollectionManager, this.cluster, - () => this.elasticsearchClient, + () => this.telemetryElasticsearchClient, { maxBucketSize: config.ui.max_bucket_size, } @@ -241,7 +241,7 @@ export class Plugin { // The new client should be inititalized with a similar config to `this.cluster` but, since we're not using // the new client in Monitoring Telemetry collection yet, setting the local client allos progress for now. // We will update the client in a follow up PR. - this.elasticsearchClient = elasticsearch.client; + this.telemetryElasticsearchClient = elasticsearch.client; } stop() { From 12ca4616bdbb274cd6ca8ea21d6d15adb262da07 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 23 Sep 2020 16:30:04 -0700 Subject: [PATCH 37/40] Reverts back to using callCluster to get the cluster info in stats route --- src/plugins/usage_collection/server/routes/stats.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/plugins/usage_collection/server/routes/stats.ts b/src/plugins/usage_collection/server/routes/stats.ts index 1bfe811e87b8fd..ef5da2eb11ba6a 100644 --- a/src/plugins/usage_collection/server/routes/stats.ts +++ b/src/plugins/usage_collection/server/routes/stats.ts @@ -70,9 +70,8 @@ export function registerStatsRoute({ return collectorSet.toObject(usage); }; - const getClusterUuid = async (esClient: ElasticsearchClient): Promise => { - const result = await esClient.info<{ cluster_uuid: string }>(); - const { cluster_uuid: uuid } = result.body; + const getClusterUuid = async (callCluster: LegacyAPICaller): Promise => { + const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid' }); return uuid; }; @@ -109,7 +108,7 @@ export function registerStatsRoute({ } const usagePromise = shouldGetUsage ? getUsage(callCluster, esClient) : Promise.resolve({}); - const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(esClient)]); + const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(callCluster)]); let modifiedUsage = usage; if (isLegacy) { From 94d06d587bf6ca26bad91501ab8f5d3f1945f55f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Wed, 23 Sep 2020 17:23:32 -0700 Subject: [PATCH 38/40] refactors get_local_license --- .../telemetry_collection/get_local_license.ts | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index 47b7962d4178bc..8bc99832fe9b0a 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -22,6 +22,14 @@ import { ElasticsearchClient } from 'src/core/server'; let cachedLicense: ESLicense | undefined; +async function fetchLicense(esClient: ElasticsearchClient, local: boolean) { + const { body } = await esClient.license.get({ + local, + // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. + accept_enterprise: true, + }); + return body.license; +} /** * Get the cluster's license from the connected node. * @@ -32,43 +40,29 @@ let cachedLicense: ESLicense | undefined; * In OSS we'll get a 400 response using the nwe client and need to catch that error too */ async function getLicenseFromLocalOrMaster(esClient: ElasticsearchClient) { - let response; - try { - // TODO: extract the call into it's own function that accepts the flag for local - // Fetching the license from the local node is cheaper than getting it from the master node and good enough - const { body } = await esClient.license.get<{ license: ESLicense }>({ - local: true, - accept_enterprise: true, - }); - cachedLicense = body.license; - response = body.license; - } catch (err) { - // if there is an error, try to get the license from the master node: + // Fetching the license from the local node is cheaper than getting it from the master node and good enough + const { license } = await fetchLicense(esClient, true).catch(async (err) => { if (cachedLicense) { try { - const { body } = await esClient.license.get<{ license: ESLicense }>({ - local: false, - accept_enterprise: true, - }); - cachedLicense = body.license; - response = body.license; + // Fallback to the master node's license info + const response = await fetchLicense(esClient, false); + return response; } catch (masterError) { if ([400, 404].includes(masterError.statusCode)) { - // the master node doesn't have a license and we assume there isn't a license + // If the master node does not have a license, we can assume there is no license cachedLicense = undefined; - response = undefined; } else { throw err; } } } - if ([400, 404].includes(err.statusCode)) { - cachedLicense = undefined; - } else { - throw err; - } + return { license: void 0 }; + }); + + if (license) { + cachedLicense = license; } - return response; + return license; } export const getLocalLicense: LicenseGetter = async (clustersDetails, { esClient }) => { From a572d75eeafabd8008c75c5e995b6f13cde77265 Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 24 Sep 2020 09:50:34 -0700 Subject: [PATCH 39/40] Returns license --- .../server/telemetry_collection/get_local_license.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index 8bc99832fe9b0a..879416cda62fc4 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -28,19 +28,19 @@ async function fetchLicense(esClient: ElasticsearchClient, local: boolean) { // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. accept_enterprise: true, }); - return body.license; + return body; } /** * Get the cluster's license from the connected node. * - * This is the equivalent of GET /_license?local=true&accept_enterprise=true . + * This is the equivalent of GET /_license?local=true&accept_enterprise=true. * * Like any X-Pack related API, X-Pack must installed for this to work. * - * In OSS we'll get a 400 response using the nwe client and need to catch that error too + * In OSS we'll get a 400 response using the new elasticsearch client. */ async function getLicenseFromLocalOrMaster(esClient: ElasticsearchClient) { - // Fetching the license from the local node is cheaper than getting it from the master node and good enough + // Fetching the local license is cheaper than getting it from the master node and good enough const { license } = await fetchLicense(esClient, true).catch(async (err) => { if (cachedLicense) { try { From 06e3b79f93de8e01e6e56b1a8472a2bd9b99b43f Mon Sep 17 00:00:00 2001 From: Christiane Heiligers Date: Thu, 24 Sep 2020 10:49:24 -0700 Subject: [PATCH 40/40] cleans up nodes usage API call --- .../server/telemetry_collection/get_nodes_usage.test.ts | 4 ++-- .../telemetry/server/telemetry_collection/get_nodes_usage.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts index 8cbd1793f6bcca..acf403ba254477 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts @@ -52,12 +52,12 @@ describe('get_nodes_usage', () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; esClient.nodes.usage.mockImplementationOnce( // @ts-ignore - async (_params = { metric: '_all', timeout: TIMEOUT }) => { + async (_params = { timeout: TIMEOUT }) => { return response; } ); const item = await getNodesUsage(esClient); - expect(esClient.nodes.usage).toHaveBeenCalledWith({ metric: '_all', timeout: TIMEOUT }); + expect(esClient.nodes.usage).toHaveBeenCalledWith({ timeout: TIMEOUT }); expect(item).toStrictEqual({ nodes: [ { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index 170e69b511d255..959840d0020a20 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -57,7 +57,6 @@ export async function fetchNodesUsage( esClient: ElasticsearchClient ): Promise { const { body } = await esClient.nodes.usage({ - metric: '_all', timeout: TIMEOUT, }); return body;