From d512b5aba4fe7b1848dcebb2ed19c2530e56590b Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Thu, 24 Sep 2020 18:32:55 -0700 Subject: [PATCH] Adds new elasticsearch client to telemetry plugin (#78046) Co-authored-by: Elastic Machine --- src/plugins/data/server/server.api.md | 1 + ...emetry_application_usage_collector.test.ts | 15 +- src/plugins/telemetry/server/plugin.ts | 17 +- .../__tests__/get_cluster_info.js | 39 --- .../__tests__/get_cluster_stats.js | 49 ---- .../__tests__/get_local_stats.js | 267 ------------------ .../get_cluster_info.test.ts | 57 ++++ .../telemetry_collection/get_cluster_info.ts | 24 +- .../get_cluster_stats.test.ts | 44 +++ .../telemetry_collection/get_cluster_stats.ts | 16 +- .../get_data_telemetry.test.ts | 89 +++--- .../get_data_telemetry/get_data_telemetry.ts | 60 ++-- .../get_data_telemetry/index.ts | 4 +- .../server/telemetry_collection/get_kibana.ts | 7 +- .../telemetry_collection/get_local_license.ts | 38 ++- .../get_local_stats.test.ts | 259 +++++++++++++++++ .../telemetry_collection/get_local_stats.ts | 25 +- .../get_nodes_usage.test.ts | 59 ++-- .../telemetry_collection/get_nodes_usage.ts | 20 +- .../register_collection.ts | 5 +- .../server/plugin.ts | 98 ++++--- .../server/types.ts | 14 +- src/plugins/usage_collection/README.md | 8 +- .../server/collector/collector.ts | 4 +- .../server/collector/collector_set.test.ts | 13 +- .../server/collector/collector_set.ts | 12 +- .../usage_collection/server/routes/stats.ts | 11 +- x-pack/plugins/monitoring/server/plugin.ts | 21 +- .../get_all_stats.test.ts | 3 + .../telemetry_collection/get_all_stats.ts | 3 +- .../get_cluster_uuids.test.ts | 6 +- .../register_monitoring_collection.ts | 4 +- .../server/plugin.ts | 14 +- .../__tests__/get_xpack.js | 46 --- .../get_stats_with_xpack.test.ts | 91 +++--- .../get_stats_with_xpack.ts | 4 +- .../telemetry_collection/get_xpack.test.ts | 19 ++ .../server/telemetry_collection/get_xpack.ts | 13 +- 38 files changed, 776 insertions(+), 703 deletions(-) delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts delete mode 100644 x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js create mode 100644 x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index f465ece697a70..9cf7234c4a9ff 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -44,6 +44,7 @@ import { DeleteDocumentParams } from 'elasticsearch'; import { DeleteScriptParams } from 'elasticsearch'; import { DeleteTemplateParams } from 'elasticsearch'; import { Duration } from 'moment'; +import { ElasticsearchClient as ElasticsearchClient_2 } from 'kibana/server'; import { Ensure } from '@kbn/utility-types'; import { EnvironmentMode } from '@kbn/config'; import { ErrorToastOptions } from 'src/core/public/notifications'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts index 709736a37d802..23a77c2d4c288 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts @@ -17,7 +17,11 @@ * under the License. */ -import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks'; +import { + savedObjectsRepositoryMock, + loggingSystemMock, + elasticsearchServiceMock, +} from '../../../../../core/server/mocks'; import { CollectorOptions, createUsageCollectionSetupMock, @@ -50,6 +54,7 @@ describe('telemetry_application_usage', () => { const getUsageCollector = jest.fn(); const registerType = jest.fn(); const callCluster = jest.fn(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; beforeAll(() => registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector) @@ -62,7 +67,7 @@ describe('telemetry_application_usage', () => { test('if no savedObjectClient initialised, return undefined', async () => { expect(collector.isReady()).toBe(false); - expect(await collector.fetch(callCluster)).toBeUndefined(); + expect(await collector.fetch(callCluster, esClient)).toBeUndefined(); jest.runTimersToTime(ROLL_INDICES_START); }); @@ -80,7 +85,7 @@ describe('telemetry_application_usage', () => { jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run expect(collector.isReady()).toBe(true); - expect(await collector.fetch(callCluster)).toStrictEqual({}); + expect(await collector.fetch(callCluster, esClient)).toStrictEqual({}); expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled(); }); @@ -137,7 +142,7 @@ describe('telemetry_application_usage', () => { jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run - expect(await collector.fetch(callCluster)).toStrictEqual({ + expect(await collector.fetch(callCluster, esClient)).toStrictEqual({ appId: { clicks_total: total + 1 + 10, clicks_7_days: total + 1, @@ -197,7 +202,7 @@ describe('telemetry_application_usage', () => { getUsageCollector.mockImplementation(() => savedObjectClient); - expect(await collector.fetch(callCluster)).toStrictEqual({ + expect(await collector.fetch(callCluster, esClient)).toStrictEqual({ appId: { clicks_total: 1, clicks_7_days: 0, diff --git a/src/plugins/telemetry/server/plugin.ts b/src/plugins/telemetry/server/plugin.ts index 005c5f96d98d0..dfbbe3355e69c 100644 --- a/src/plugins/telemetry/server/plugin.ts +++ b/src/plugins/telemetry/server/plugin.ts @@ -34,6 +34,7 @@ import { SavedObjectsClient, Plugin, Logger, + IClusterClient, } from '../../../core/server'; import { registerRoutes } from './routes'; import { registerCollection } from './telemetry_collection'; @@ -83,6 +84,7 @@ export class TelemetryPlugin implements Plugin) { this.logger = initializerContext.logger.get(); @@ -102,8 +104,11 @@ export class TelemetryPlugin implements Plugin this.elasticsearchClient + ); const router = http.createRouter(); registerRoutes({ @@ -126,14 +131,12 @@ export class TelemetryPlugin implements Plugin { - const { savedObjects, uiSettings } = core; + public async start(core: CoreStart, { telemetryCollectionManager }: TelemetryPluginsDepsStart) { + const { savedObjects, uiSettings, elasticsearch } = core; this.savedObjectsClient = savedObjects.createInternalRepository(); const savedObjectsClient = new SavedObjectsClient(this.savedObjectsClient); this.uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient); + this.elasticsearchClient = elasticsearch.client; try { await handleOldSettings(savedObjectsClient, this.uiSettingsClient); diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js deleted file mode 100644 index fe83b76cd1158..0000000000000 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; - -import { getClusterInfo } from '../get_cluster_info'; - -export function mockGetClusterInfo(callCluster, clusterInfo, req) { - callCluster.withArgs(req, 'info').returns(clusterInfo); - callCluster.withArgs('info').returns(clusterInfo); -} - -describe('get_cluster_info', () => { - it('uses callCluster to get info API', () => { - const callCluster = sinon.stub(); - const response = Promise.resolve({}); - - mockGetClusterInfo(callCluster, response); - - expect(getClusterInfo(callCluster)).to.be(response); - }); -}); diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js deleted file mode 100644 index d1354608385f6..0000000000000 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; - -import { TIMEOUT } from '../constants'; -import { getClusterStats } from '../get_cluster_stats'; - -export function mockGetClusterStats(callCluster, clusterStats, req) { - callCluster - .withArgs(req, 'cluster.stats', { - timeout: TIMEOUT, - }) - .returns(clusterStats); - - callCluster - .withArgs('cluster.stats', { - timeout: TIMEOUT, - }) - .returns(clusterStats); -} - -describe.skip('get_cluster_stats', () => { - it('uses callCluster to get cluster.stats API', async () => { - const callCluster = sinon.stub(); - const response = Promise.resolve({}); - - mockGetClusterStats(callCluster, response); - - expect(getClusterStats(callCluster)).to.be(response); - }); -}); diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js deleted file mode 100644 index 8541745faea3b..0000000000000 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; -import { merge, omit } from 'lodash'; - -import { TIMEOUT } from '../constants'; -import { mockGetClusterInfo } from './get_cluster_info'; -import { mockGetClusterStats } from './get_cluster_stats'; - -import { getLocalStats, handleLocalStats } from '../get_local_stats'; - -const mockUsageCollection = (kibanaUsage = {}) => ({ - bulkFetch: () => kibanaUsage, - toObject: (data) => data, -}); - -const getMockServer = (getCluster = sinon.stub()) => ({ - log(tags, message) { - console.log({ tags, message }); - }, - config() { - return { - get(item) { - switch (item) { - case 'pkg.version': - return '8675309-snapshot'; - default: - throw Error(`unexpected config.get('${item}') received.`); - } - }, - }; - }, - plugins: { - elasticsearch: { getCluster }, - }, -}); -function mockGetNodesUsage(callCluster, nodesUsage, req) { - callCluster - .withArgs( - req, - { - method: 'GET', - path: '/_nodes/usage', - query: { - timeout: TIMEOUT, - }, - }, - 'transport.request' - ) - .returns(nodesUsage); -} - -function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) { - mockGetClusterInfo(callCluster, clusterInfo, req); - mockGetClusterStats(callCluster, clusterStats, req); - mockGetNodesUsage(callCluster, nodesUsage, req); -} - -describe('get_local_stats', () => { - const clusterUuid = 'abc123'; - const clusterName = 'my-cool-cluster'; - const version = '2.3.4'; - const clusterInfo = { - cluster_uuid: clusterUuid, - cluster_name: clusterName, - version: { - number: version, - }, - }; - const nodesUsage = [ - { - node_id: 'some_node_id', - timestamp: 1588617023177, - since: 1588616945163, - rest_actions: { - nodes_usage_action: 1, - create_index_action: 1, - document_get_action: 1, - search_action: 19, - nodes_info_action: 36, - }, - aggregations: { - terms: { - bytes: 2, - }, - scripted_metric: { - other: 7, - }, - }, - }, - ]; - const clusterStats = { - _nodes: { failed: 123 }, - cluster_name: 'real-cool', - indices: { totally: 456 }, - nodes: { yup: 'abc' }, - random: 123, - }; - - const kibana = { - kibana: { - great: 'googlymoogly', - versions: [{ version: '8675309', count: 1 }], - }, - kibana_stats: { - os: { - platform: 'rocky', - platformRelease: 'iv', - }, - }, - localization: { - locale: 'en', - labelsCount: 0, - integrities: {}, - }, - sun: { chances: 5 }, - clouds: { chances: 95 }, - rain: { chances: 2 }, - snow: { chances: 0 }, - }; - - const clusterStatsWithNodesUsage = { - ...clusterStats, - nodes: merge(clusterStats.nodes, { usage: nodesUsage }), - }; - const combinedStatsResult = { - collection: 'local', - cluster_uuid: clusterUuid, - cluster_name: clusterName, - version, - cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'), - stack_stats: { - kibana: { - great: 'googlymoogly', - count: 1, - indices: 1, - os: { - platforms: [{ platform: 'rocky', count: 1 }], - platformReleases: [{ platformRelease: 'iv', count: 1 }], - }, - versions: [{ version: '8675309', count: 1 }], - plugins: { - localization: { - locale: 'en', - labelsCount: 0, - integrities: {}, - }, - sun: { chances: 5 }, - clouds: { chances: 95 }, - rain: { chances: 2 }, - snow: { chances: 0 }, - }, - }, - }, - }; - - const context = { - logger: console, - version: '8.0.0', - }; - - describe('handleLocalStats', () => { - it('returns expected object without xpack and kibana data', () => { - const result = handleLocalStats( - clusterInfo, - clusterStatsWithNodesUsage, - void 0, - void 0, - context - ); - expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); - expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); - expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); - expect(result.version).to.be('2.3.4'); - expect(result.collection).to.be('local'); - expect(result.license).to.be(undefined); - expect(result.stack_stats).to.eql({ kibana: undefined, data: undefined }); - }); - - it('returns expected object with xpack', () => { - const result = handleLocalStats( - clusterInfo, - clusterStatsWithNodesUsage, - void 0, - void 0, - context - ); - const { stack_stats: stack, ...cluster } = result; - expect(cluster.collection).to.be(combinedStatsResult.collection); - expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid); - expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name); - expect(stack.kibana).to.be(undefined); // not mocked for this test - expect(stack.data).to.be(undefined); // not mocked for this test - - expect(cluster.version).to.eql(combinedStatsResult.version); - expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats); - expect(cluster.license).to.eql(combinedStatsResult.license); - expect(stack.xpack).to.eql(combinedStatsResult.stack_stats.xpack); - }); - }); - - describe.skip('getLocalStats', () => { - it('returns expected object without xpack data when X-Pack fails to respond', async () => { - const callClusterUsageFailed = sinon.stub(); - const usageCollection = mockUsageCollection(); - mockGetLocalStats( - callClusterUsageFailed, - Promise.resolve(clusterInfo), - Promise.resolve(clusterStats), - Promise.resolve(nodesUsage) - ); - const result = await getLocalStats([], { - server: getMockServer(), - callCluster: callClusterUsageFailed, - usageCollection, - }); - expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); - expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); - expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); - expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes); - expect(result.version).to.be('2.3.4'); - expect(result.collection).to.be('local'); - - // license and xpack usage info come from the same cluster call - expect(result.license).to.be(undefined); - expect(result.stack_stats.xpack).to.be(undefined); - }); - - it('returns expected object with xpack and kibana data', async () => { - const callCluster = sinon.stub(); - const usageCollection = mockUsageCollection(kibana); - mockGetLocalStats( - callCluster, - Promise.resolve(clusterInfo), - Promise.resolve(clusterStats), - Promise.resolve(nodesUsage) - ); - - const result = await getLocalStats([], { - server: getMockServer(callCluster), - usageCollection, - callCluster, - }); - - expect(result.stack_stats.xpack).to.eql(combinedStatsResult.stack_stats.xpack); - expect(result.stack_stats.kibana).to.eql(combinedStatsResult.stack_stats.kibana); - }); - }); -}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts new file mode 100644 index 0000000000000..459b18d252e17 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { getClusterInfo } from './get_cluster_info'; + +export function mockGetClusterInfo(clusterInfo: any) { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.info + // @ts-ignore we only care about the response body + .mockResolvedValue( + // @ts-ignore we only care about the response body + { + body: { ...clusterInfo }, + } + ); + return esClient; +} + +describe('get_cluster_info using the elasticsearch client', () => { + it('uses the esClient to get info API', async () => { + const clusterInfo = { + cluster_uuid: '1234', + cluster_name: 'testCluster', + version: { + number: '7.9.2', + build_flavor: 'default', + build_type: 'docker', + build_hash: 'b5ca9c58fb664ca8bf', + build_date: '2020-07-21T16:40:44.668009Z', + build_snapshot: false, + lucene_version: '8.5.1', + minimum_wire_compatibility_version: '6.8.0', + minimum_index_compatibility_version: '6.0.0-beta1', + }, + }; + const esClient = mockGetClusterInfo(clusterInfo); + + expect(await getClusterInfo(esClient)).toStrictEqual(clusterInfo); + }); +}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts index 4a33356ee9761..407f3325c3a9f 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; // This can be removed when the ES client improves the types export interface ESClusterInfo { @@ -25,24 +25,24 @@ export interface ESClusterInfo { cluster_name: string; version: { number: string; - build_flavor: string; - build_type: string; - build_hash: string; - build_date: string; + build_flavor?: string; + build_type?: string; + build_hash?: string; + build_date?: string; build_snapshot?: boolean; - lucene_version: string; - minimum_wire_compatibility_version: string; - minimum_index_compatibility_version: string; + lucene_version?: string; + minimum_wire_compatibility_version?: string; + minimum_index_compatibility_version?: string; }; } - /** * Get the cluster info from the connected cluster. * * This is the equivalent to GET / * - * @param {function} callCluster The callWithInternalUser handler (exposed for testing) + * @param {function} esClient The asInternalUser handler (exposed for testing) */ -export function getClusterInfo(callCluster: LegacyAPICaller) { - return callCluster('info'); +export async function getClusterInfo(esClient: ElasticsearchClient) { + const { body } = await esClient.info(); + return body; } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts new file mode 100644 index 0000000000000..81551c0c4d93d --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { getClusterStats } from './get_cluster_stats'; +import { TIMEOUT } from './constants'; + +export function mockGetClusterStats(clusterStats: any) { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.cluster.stats.mockResolvedValue(clusterStats); + return esClient; +} + +describe('get_cluster_stats', () => { + it('uses the esClient to get the response from the `cluster.stats` API', async () => { + const response = Promise.resolve({ body: { cluster_uuid: '1234' } }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.cluster.stats.mockImplementationOnce( + // @ts-ignore the method only cares about the response body + async (_params = { timeout: TIMEOUT }) => { + return response; + } + ); + const result = getClusterStats(esClient); + expect(esClient.cluster.stats).toHaveBeenCalledWith({ timeout: TIMEOUT }); + expect(result).toStrictEqual(response); + }); +}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts index d7c0110a99c6f..d2a64e4878679 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts @@ -18,23 +18,23 @@ */ import { ClusterDetailsGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; /** * Get the cluster stats from the connected cluster. * * This is the equivalent to GET /_cluster/stats?timeout=30s. */ -export async function getClusterStats(callCluster: LegacyAPICaller) { - return await callCluster('cluster.stats', { - timeout: TIMEOUT, - }); +export async function getClusterStats(esClient: ElasticsearchClient) { + const { body } = await esClient.cluster.stats({ timeout: TIMEOUT }); + return body; } /** * Get the cluster uuids from the connected cluster. */ -export const getClusterUuids: ClusterDetailsGetter = async ({ callCluster }) => { - const result = await getClusterStats(callCluster); - return [{ clusterUuid: result.cluster_uuid }]; +export const getClusterUuids: ClusterDetailsGetter = async ({ esClient }) => { + const { body } = await esClient.cluster.stats({ timeout: TIMEOUT }); + + return [{ clusterUuid: body.cluster_uuid }]; }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts index dee718decdc1f..bb5eb7f6b726d 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts @@ -19,6 +19,7 @@ import { buildDataTelemetryPayload, getDataTelemetry } from './get_data_telemetry'; import { DATA_DATASETS_INDEX_PATTERNS, DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from './constants'; +import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks'; describe('get_data_telemetry', () => { describe('DATA_DATASETS_INDEX_PATTERNS', () => { @@ -195,13 +196,15 @@ describe('get_data_telemetry', () => { describe('getDataTelemetry', () => { test('it returns the base payload (all 0s) because no indices are found', async () => { - const callCluster = mockCallCluster(); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]); + const esClient = mockEsClient(); + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]); + expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1); + expect(esClient.indices.stats).toHaveBeenCalledTimes(1); }); test('can only see the index mappings, but not the stats', async () => { - const callCluster = mockCallCluster(['filebeat-12314']); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + const esClient = mockEsClient(['filebeat-12314']); + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([ { pattern_name: 'filebeat', shipper: 'filebeat', @@ -209,10 +212,12 @@ describe('get_data_telemetry', () => { ecs_index_count: 0, }, ]); + expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1); + expect(esClient.indices.stats).toHaveBeenCalledTimes(1); }); test('can see the mappings and the stats', async () => { - const callCluster = mockCallCluster( + const esClient = mockEsClient( ['filebeat-12314'], { isECS: true }, { @@ -221,7 +226,7 @@ describe('get_data_telemetry', () => { }, } ); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([ { pattern_name: 'filebeat', shipper: 'filebeat', @@ -234,7 +239,7 @@ describe('get_data_telemetry', () => { }); test('find an index that does not match any index pattern but has mappings metadata', async () => { - const callCluster = mockCallCluster( + const esClient = mockEsClient( ['cannot_match_anything'], { isECS: true, dataStreamType: 'traces', shipper: 'my-beat' }, { @@ -245,7 +250,7 @@ describe('get_data_telemetry', () => { }, } ); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([ { data_stream: { dataset: undefined, type: 'traces' }, shipper: 'my-beat', @@ -258,45 +263,51 @@ describe('get_data_telemetry', () => { }); test('return empty array when there is an error', async () => { - const callCluster = jest.fn().mockRejectedValue(new Error('Something went terribly wrong')); - await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.indices.getMapping.mockRejectedValue(new Error('Something went terribly wrong')); + esClient.indices.stats.mockRejectedValue(new Error('Something went terribly wrong')); + await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]); }); }); }); - -function mockCallCluster( - indicesMappings: string[] = [], +function mockEsClient( + indicesMappings: string[] = [], // an array of `indices` to get mappings from. { isECS = false, dataStreamDataset = '', dataStreamType = '', shipper = '' } = {}, indexStats: any = {} ) { - return jest.fn().mockImplementation(async (method: string, opts: any) => { - if (method === 'indices.getMapping') { - return Object.fromEntries( - indicesMappings.map((index) => [ - index, - { - mappings: { - ...(shipper && { _meta: { beat: shipper } }), - properties: { - ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }), - ...((dataStreamType || dataStreamDataset) && { - data_stream: { - properties: { - ...(dataStreamDataset && { - dataset: { type: 'constant_keyword', value: dataStreamDataset }, - }), - ...(dataStreamType && { - type: { type: 'constant_keyword', value: dataStreamType }, - }), - }, + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // @ts-ignore + esClient.indices.getMapping.mockImplementationOnce(async () => { + const body = Object.fromEntries( + indicesMappings.map((index) => [ + index, + { + mappings: { + ...(shipper && { _meta: { beat: shipper } }), + properties: { + ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }), + ...((dataStreamType || dataStreamDataset) && { + data_stream: { + properties: { + ...(dataStreamDataset && { + dataset: { type: 'constant_keyword', value: dataStreamDataset }, + }), + ...(dataStreamType && { + type: { type: 'constant_keyword', value: dataStreamType }, + }), }, - }), - }, + }, + }), }, }, - ]) - ); - } - return indexStats; + }, + ]) + ); + return { body }; + }); + // @ts-ignore + esClient.indices.stats.mockImplementationOnce(async () => { + return { body: indexStats }; }); + return esClient; } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts index f4734dde251cc..67769793cbfdf 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ +import { ElasticsearchClient } from 'src/core/server'; -import { LegacyAPICaller } from 'kibana/server'; import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE, DataPatternName, @@ -224,42 +224,50 @@ interface IndexMappings { }; } -export async function getDataTelemetry(callCluster: LegacyAPICaller) { +export async function getDataTelemetry(esClient: ElasticsearchClient) { try { const index = [ ...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern), '*-*-*', // Include data-streams aliases `{type}-{dataset}-{namespace}` ]; - const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([ + const indexMappingsParams: { index: string; filter_path: string[] } = { // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value - callCluster('indices.getMapping', { - index: '*', // Request all indices because filter_path already filters out the indices without any of those fields - filterPath: [ - // _meta.beat tells the shipper - '*.mappings._meta.beat', - // _meta.package.name tells the Ingest Manager's package - '*.mappings._meta.package.name', - // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it - '*.mappings._meta.managed_by', - // Does it have `ecs.version` in the mappings? => It follows the ECS conventions - '*.mappings.properties.ecs.properties.version.type', + index: '*', // Request all indices because filter_path already filters out the indices without any of those fields + filter_path: [ + // _meta.beat tells the shipper + '*.mappings._meta.beat', + // _meta.package.name tells the Ingest Manager's package + '*.mappings._meta.package.name', + // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it + '*.mappings._meta.managed_by', + // Does it have `ecs.version` in the mappings? => It follows the ECS conventions + '*.mappings.properties.ecs.properties.version.type', - // If `data_stream.type` is a `constant_keyword`, it can be reported as a type - '*.mappings.properties.data_stream.properties.type.value', - // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset - '*.mappings.properties.data_stream.properties.dataset.value', - ], - }), + // If `data_stream.type` is a `constant_keyword`, it can be reported as a type + '*.mappings.properties.data_stream.properties.type.value', + // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset + '*.mappings.properties.data_stream.properties.dataset.value', + ], + }; + const indicesStatsParams: { + index: string | string[] | undefined; + level: 'cluster' | 'indices' | 'shards' | undefined; + metric: string[]; + filter_path: string[]; + } = { // GET /_stats/docs,store?level=indices&filter_path=indices.*.total - callCluster('indices.stats', { - index, - level: 'indices', - metric: ['docs', 'store'], - filterPath: ['indices.*.total'], - }), + index, + level: 'indices', + metric: ['docs', 'store'], + filter_path: ['indices.*.total'], + }; + const [{ body: indexMappings }, { body: indexStats }] = await Promise.all([ + esClient.indices.getMapping(indexMappingsParams), + esClient.indices.stats(indicesStatsParams), ]); const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices }); + const indices = indexNames.map((name) => { const baseIndexInfo = { name, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts index d056d1c9f299f..0e2ab98a24cba 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts @@ -20,8 +20,8 @@ export { DATA_TELEMETRY_ID } from './constants'; export { - DataTelemetryIndex, - DataTelemetryPayload, getDataTelemetry, buildDataTelemetryPayload, + DataTelemetryPayload, + DataTelemetryIndex, } from './get_data_telemetry'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts index 5d27774a630a5..0ef9815a4eadb 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts @@ -21,6 +21,7 @@ import { omit } from 'lodash'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { LegacyAPICaller } from 'kibana/server'; import { StatsCollectionContext } from 'src/plugins/telemetry_collection_manager/server'; +import { ElasticsearchClient } from 'src/core/server'; export interface KibanaUsageStats { kibana: { @@ -48,7 +49,6 @@ export function handleKibanaStats( logger.warn('No Kibana stats returned from usage collectors'); return; } - const { kibana, kibana_stats: kibanaStats, ...plugins } = response; const os = { @@ -83,8 +83,9 @@ export function handleKibanaStats( export async function getKibana( usageCollection: UsageCollectionSetup, - callWithInternalUser: LegacyAPICaller + callWithInternalUser: LegacyAPICaller, + asInternalUser: ElasticsearchClient ): Promise { - const usage = await usageCollection.bulkFetch(callWithInternalUser); + const usage = await usageCollection.bulkFetch(callWithInternalUser, asInternalUser); return usageCollection.toObject(usage); } diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts index d41904c6d8e0e..879416cda62fc 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts @@ -17,44 +17,41 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manager/server'; +import { ElasticsearchClient } from 'src/core/server'; let cachedLicense: ESLicense | undefined; -function fetchLicense(callCluster: LegacyAPICaller, local: boolean) { - return callCluster<{ license: ESLicense }>('transport.request', { - method: 'GET', - path: '/_license', - query: { - local, - // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. - accept_enterprise: 'true', - }, +async function fetchLicense(esClient: ElasticsearchClient, local: boolean) { + const { body } = await esClient.license.get({ + local, + // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. + accept_enterprise: true, }); + return body; } - /** * Get the cluster's license from the connected node. * - * This is the equivalent of GET /_license?local=true . + * This is the equivalent of GET /_license?local=true&accept_enterprise=true. * * Like any X-Pack related API, X-Pack must installed for this to work. + * + * In OSS we'll get a 400 response using the new elasticsearch client. */ -async function getLicenseFromLocalOrMaster(callCluster: LegacyAPICaller) { - // Fetching the local license is cheaper than getting it from the master and good enough - const { license } = await fetchLicense(callCluster, true).catch(async (err) => { +async function getLicenseFromLocalOrMaster(esClient: ElasticsearchClient) { + // Fetching the local license is cheaper than getting it from the master node and good enough + const { license } = await fetchLicense(esClient, true).catch(async (err) => { if (cachedLicense) { try { // Fallback to the master node's license info - const response = await fetchLicense(callCluster, false); + const response = await fetchLicense(esClient, false); return response; } catch (masterError) { - if (masterError.statusCode === 404) { + if ([400, 404].includes(masterError.statusCode)) { // If the master node does not have a license, we can assume there is no license cachedLicense = undefined; } else { - // Any other errors from the master node, throw and do not send any telemetry throw err; } } @@ -68,9 +65,8 @@ async function getLicenseFromLocalOrMaster(callCluster: LegacyAPICaller) { return license; } -export const getLocalLicense: LicenseGetter = async (clustersDetails, { callCluster }) => { - const license = await getLicenseFromLocalOrMaster(callCluster); - +export const getLocalLicense: LicenseGetter = async (clustersDetails, { esClient }) => { + const license = await getLicenseFromLocalOrMaster(esClient); // It should be called only with 1 cluster element in the clustersDetails array, but doing reduce just in case. return clustersDetails.reduce((acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), {}); }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts new file mode 100644 index 0000000000000..0c8b0b249f7d1 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts @@ -0,0 +1,259 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { merge, omit } from 'lodash'; + +import { getLocalStats, handleLocalStats } from './get_local_stats'; +import { usageCollectionPluginMock } from '../../../usage_collection/server/mocks'; +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; + +function mockUsageCollection(kibanaUsage = {}) { + const usageCollection = usageCollectionPluginMock.createSetupContract(); + usageCollection.bulkFetch = jest.fn().mockResolvedValue(kibanaUsage); + usageCollection.toObject = jest.fn().mockImplementation((data: any) => data); + return usageCollection; +} +// set up successful call mocks for info, cluster stats, nodes usage and data telemetry +function mockGetLocalStats(clusterInfo: any, clusterStats: any) { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.info + // @ts-ignore we only care about the response body + .mockResolvedValue( + // @ts-ignore we only care about the response body + { + body: { ...clusterInfo }, + } + ); + esClient.cluster.stats + // @ts-ignore we only care about the response body + .mockResolvedValue({ body: { ...clusterStats } }); + esClient.nodes.usage.mockResolvedValue( + // @ts-ignore we only care about the response body + { + body: { + cluster_name: 'testCluster', + nodes: { + some_node_id: { + timestamp: 1588617023177, + since: 1588616945163, + rest_actions: { + nodes_usage_action: 1, + create_index_action: 1, + document_get_action: 1, + search_action: 19, + nodes_info_action: 36, + }, + aggregations: { + terms: { + bytes: 2, + }, + scripted_metric: { + other: 7, + }, + }, + }, + }, + }, + } + ); + // @ts-ignore we only care about the response body + esClient.indices.getMapping.mockResolvedValue({ body: { mappings: {} } }); + // @ts-ignore we only care about the response body + esClient.indices.stats.mockResolvedValue({ body: { indices: {} } }); + return esClient; +} + +describe('get_local_stats', () => { + const clusterUuid = 'abc123'; + const clusterName = 'my-cool-cluster'; + const version = '2.3.4'; + const clusterInfo = { + cluster_uuid: clusterUuid, + cluster_name: clusterName, + version: { number: version }, + }; + const nodesUsage = [ + { + node_id: 'some_node_id', + timestamp: 1588617023177, + since: 1588616945163, + rest_actions: { + nodes_usage_action: 1, + create_index_action: 1, + document_get_action: 1, + search_action: 19, + nodes_info_action: 36, + }, + aggregations: { + terms: { + bytes: 2, + }, + scripted_metric: { + other: 7, + }, + }, + }, + ]; + const clusterStats = { + _nodes: { failed: 123 }, + cluster_name: 'real-cool', + indices: { totally: 456 }, + nodes: { yup: 'abc' }, + random: 123, + }; + + const kibana = { + kibana: { + great: 'googlymoogly', + versions: [{ version: '8675309', count: 1 }], + }, + kibana_stats: { + os: { + platform: 'rocky', + platformRelease: 'iv', + }, + }, + localization: { + locale: 'en', + labelsCount: 0, + integrities: {}, + }, + sun: { chances: 5 }, + clouds: { chances: 95 }, + rain: { chances: 2 }, + snow: { chances: 0 }, + }; + + const clusterStatsWithNodesUsage = { + ...clusterStats, + nodes: merge(clusterStats.nodes, { usage: { nodes: nodesUsage } }), + }; + + const combinedStatsResult = { + collection: 'local', + cluster_uuid: clusterUuid, + cluster_name: clusterName, + version, + cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'), + stack_stats: { + kibana: { + great: 'googlymoogly', + count: 1, + indices: 1, + os: { + platforms: [{ platform: 'rocky', count: 1 }], + platformReleases: [{ platformRelease: 'iv', count: 1 }], + }, + versions: [{ version: '8675309', count: 1 }], + plugins: { + localization: { + locale: 'en', + labelsCount: 0, + integrities: {}, + }, + sun: { chances: 5 }, + clouds: { chances: 95 }, + rain: { chances: 2 }, + snow: { chances: 0 }, + }, + }, + }, + }; + + const context = { + logger: console, + version: '8.0.0', + }; + + describe('handleLocalStats', () => { + it('returns expected object without xpack or kibana data', () => { + const result = handleLocalStats( + clusterInfo, + clusterStatsWithNodesUsage, + void 0, + void 0, + context + ); + expect(result.cluster_uuid).toStrictEqual(combinedStatsResult.cluster_uuid); + expect(result.cluster_name).toStrictEqual(combinedStatsResult.cluster_name); + expect(result.cluster_stats).toStrictEqual(combinedStatsResult.cluster_stats); + expect(result.version).toEqual('2.3.4'); + expect(result.collection).toEqual('local'); + expect(Object.keys(result)).not.toContain('license'); + expect(result.stack_stats).toEqual({ kibana: undefined, data: undefined }); + }); + + it('returns expected object with xpack', () => { + const result = handleLocalStats( + clusterInfo, + clusterStatsWithNodesUsage, + void 0, + void 0, + context + ); + + const { stack_stats: stack, ...cluster } = result; + expect(cluster.collection).toBe(combinedStatsResult.collection); + expect(cluster.cluster_uuid).toBe(combinedStatsResult.cluster_uuid); + expect(cluster.cluster_name).toBe(combinedStatsResult.cluster_name); + expect(stack.kibana).toBe(undefined); // not mocked for this test + expect(stack.data).toBe(undefined); // not mocked for this test + + expect(cluster.version).toEqual(combinedStatsResult.version); + expect(cluster.cluster_stats).toEqual(combinedStatsResult.cluster_stats); + expect(Object.keys(cluster).indexOf('license')).toBeLessThan(0); + expect(Object.keys(stack).indexOf('xpack')).toBeLessThan(0); + }); + }); + + describe('getLocalStats', () => { + it('returns expected object with kibana data', async () => { + const callCluster = jest.fn(); + const usageCollection = mockUsageCollection(kibana); + const esClient = mockGetLocalStats(clusterInfo, clusterStats); + const response = await getLocalStats( + [{ clusterUuid: 'abc123' }], + { callCluster, usageCollection, esClient, start: '', end: '' }, + context + ); + const result = response[0]; + expect(result.cluster_uuid).toEqual(combinedStatsResult.cluster_uuid); + expect(result.cluster_name).toEqual(combinedStatsResult.cluster_name); + expect(result.cluster_stats).toEqual(combinedStatsResult.cluster_stats); + expect(result.cluster_stats.nodes).toEqual(combinedStatsResult.cluster_stats.nodes); + expect(result.version).toBe('2.3.4'); + expect(result.collection).toBe('local'); + expect(Object.keys(result).indexOf('license')).toBeLessThan(0); + expect(Object.keys(result.stack_stats).indexOf('xpack')).toBeLessThan(0); + }); + + it('returns an empty array when no cluster uuid is provided', async () => { + const callCluster = jest.fn(); + const usageCollection = mockUsageCollection(kibana); + const esClient = mockGetLocalStats(clusterInfo, clusterStats); + const response = await getLocalStats( + [], + { callCluster, usageCollection, esClient, start: '', end: '' }, + context + ); + expect(response).toBeDefined(); + expect(response.length).toEqual(0); + }); + }); +}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index 98c83a3394628..6244c6fac51d3 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -40,8 +40,8 @@ export function handleLocalStats( // eslint-disable-next-line @typescript-eslint/naming-convention { cluster_name, cluster_uuid, version }: ESClusterInfo, { _nodes, cluster_name: clusterName, ...clusterStats }: any, - kibana: KibanaUsageStats, - dataTelemetry: DataTelemetryPayload, + kibana: KibanaUsageStats | undefined, + dataTelemetry: DataTelemetryPayload | undefined, context: StatsCollectionContext ) { return { @@ -62,22 +62,25 @@ export type TelemetryLocalStats = ReturnType; /** * Get statistics for all products joined by Elasticsearch cluster. + * @param {Array} cluster uuids + * @param {Object} config contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end + * @param {Object} StatsCollectionContext contains logger and version (string) */ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( - clustersDetails, - config, - context + clustersDetails, // array of cluster uuid's + config, // contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end + context // StatsCollectionContext contains logger and version (string) ) => { - const { callCluster, usageCollection } = config; + const { callCluster, usageCollection, esClient } = config; return await Promise.all( clustersDetails.map(async (clustersDetail) => { const [clusterInfo, clusterStats, nodesUsage, kibana, dataTelemetry] = await Promise.all([ - getClusterInfo(callCluster), // cluster info - getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_) - getNodesUsage(callCluster), // nodes_usage info - getKibana(usageCollection, callCluster), - getDataTelemetry(callCluster), + getClusterInfo(esClient), // cluster info + getClusterStats(esClient), // cluster stats (not to be confused with cluster _state_) + getNodesUsage(esClient), // nodes_usage info + getKibana(usageCollection, callCluster, esClient), + getDataTelemetry(esClient), ]); return handleLocalStats( clusterInfo, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts index 4e4b0e11b7979..acf403ba25447 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts @@ -19,6 +19,7 @@ import { getNodesUsage } from './get_nodes_usage'; import { TIMEOUT } from './constants'; +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; const mockedNodesFetchResponse = { cluster_name: 'test cluster', @@ -44,37 +45,35 @@ const mockedNodesFetchResponse = { }, }, }; + describe('get_nodes_usage', () => { - it('calls fetchNodesUsage', async () => { - const callCluster = jest.fn(); - callCluster.mockResolvedValueOnce(mockedNodesFetchResponse); - await getNodesUsage(callCluster); - expect(callCluster).toHaveBeenCalledWith('transport.request', { - path: '/_nodes/usage', - method: 'GET', - query: { - timeout: TIMEOUT, - }, - }); - }); - it('returns a modified array of node usage data', async () => { - const callCluster = jest.fn(); - callCluster.mockResolvedValueOnce(mockedNodesFetchResponse); - const result = await getNodesUsage(callCluster); - expect(result.nodes).toEqual([ - { - aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } }, - node_id: 'some_node_id', - rest_actions: { - create_index_action: 1, - document_get_action: 1, - nodes_info_action: 36, - nodes_usage_action: 1, - search_action: 19, + it('returns a modified array of nodes usage data', async () => { + const response = Promise.resolve({ body: mockedNodesFetchResponse }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.nodes.usage.mockImplementationOnce( + // @ts-ignore + async (_params = { timeout: TIMEOUT }) => { + return response; + } + ); + const item = await getNodesUsage(esClient); + expect(esClient.nodes.usage).toHaveBeenCalledWith({ timeout: TIMEOUT }); + expect(item).toStrictEqual({ + nodes: [ + { + aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } }, + node_id: 'some_node_id', + rest_actions: { + create_index_action: 1, + document_get_action: 1, + nodes_info_action: 36, + nodes_usage_action: 1, + search_action: 19, + }, + since: 1588616945163, + timestamp: 1588617023177, }, - since: 1588616945163, - timestamp: 1588617023177, - }, - ]); + ], + }); }); }); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts index c5c110fbb4149..959840d0020a2 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; export interface NodeAggregation { @@ -44,7 +44,7 @@ export interface NodesFeatureUsageResponse { } export type NodesUsageGetter = ( - callCluster: LegacyAPICaller + esClient: ElasticsearchClient ) => Promise<{ nodes: NodeObj[] | Array<{}> }>; /** * Get the nodes usage data from the connected cluster. @@ -54,16 +54,12 @@ export type NodesUsageGetter = ( * The Nodes usage API was introduced in v6.0.0 */ export async function fetchNodesUsage( - callCluster: LegacyAPICaller + esClient: ElasticsearchClient ): Promise { - const response = await callCluster('transport.request', { - method: 'GET', - path: '/_nodes/usage', - query: { - timeout: TIMEOUT, - }, + const { body } = await esClient.nodes.usage({ + timeout: TIMEOUT, }); - return response; + return body; } /** @@ -71,8 +67,8 @@ export async function fetchNodesUsage( * @param callCluster APICaller * @returns Object containing array of modified usage information with the node_id nested within the data for that node. */ -export const getNodesUsage: NodesUsageGetter = async (callCluster) => { - const result = await fetchNodesUsage(callCluster); +export const getNodesUsage: NodesUsageGetter = async (esClient) => { + const result = await fetchNodesUsage(esClient); const transformedNodes = Object.entries(result?.nodes || {}).map(([key, value]) => ({ ...(value as NodeObj), node_id: key, diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts index 438fcadad9255..9dac4900f5f10 100644 --- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts +++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts @@ -38,16 +38,19 @@ import { ILegacyClusterClient } from 'kibana/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; +import { IClusterClient } from '../../../../../src/core/server'; import { getLocalStats } from './get_local_stats'; import { getClusterUuids } from './get_cluster_stats'; import { getLocalLicense } from './get_local_license'; export function registerCollection( telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, - esCluster: ILegacyClusterClient + esCluster: ILegacyClusterClient, + esClientGetter: () => IClusterClient | undefined ) { telemetryCollectionManager.setCollection({ esCluster, + esClientGetter, title: 'local', priority: 0, statsGetter: getLocalStats, diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 051bb3a11cb16..e54e7451a670a 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -24,6 +24,7 @@ import { CoreStart, Plugin, Logger, + IClusterClient, } from '../../../core/server'; import { @@ -86,6 +87,7 @@ export class TelemetryCollectionManagerPlugin title, priority, esCluster, + esClientGetter, statsGetter, clusterDetailsGetter, licenseGetter, @@ -105,6 +107,9 @@ export class TelemetryCollectionManagerPlugin if (!esCluster) { throw Error('esCluster name must be set for the getCluster method.'); } + if (!esClientGetter) { + throw Error('esClientGetter method not set.'); + } if (!clusterDetailsGetter) { throw Error('Cluster UUIds method is not set.'); } @@ -118,6 +123,7 @@ export class TelemetryCollectionManagerPlugin clusterDetailsGetter, esCluster, title, + esClientGetter, }); this.usageGetterMethodPriority = priority; } @@ -126,6 +132,7 @@ export class TelemetryCollectionManagerPlugin private getStatsCollectionConfig( config: StatsGetterConfig, collection: Collection, + collectionEsClient: IClusterClient, usageCollection: UsageCollectionSetup ): StatsCollectionConfig { const { start, end, request } = config; @@ -133,8 +140,11 @@ export class TelemetryCollectionManagerPlugin const callCluster = config.unencrypted ? collection.esCluster.asScoped(request).callAsCurrentUser : collection.esCluster.callAsInternalUser; - - return { callCluster, start, end, usageCollection }; + // Scope the new elasticsearch Client appropriately and pass to the stats collection config + const esClient = config.unencrypted + ? collectionEsClient.asScoped(config.request).asCurrentUser + : collectionEsClient.asInternalUser; + return { callCluster, start, end, usageCollection, esClient }; } private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) { @@ -142,27 +152,33 @@ export class TelemetryCollectionManagerPlugin return []; } for (const collection of this.collections) { - const statsCollectionConfig = this.getStatsCollectionConfig( - config, - collection, - this.usageCollection - ); - try { - const optInStats = await this.getOptInStatsForCollection( + // first fetch the client and make sure it's not undefined. + const collectionEsClient = collection.esClientGetter(); + if (collectionEsClient !== undefined) { + const statsCollectionConfig = this.getStatsCollectionConfig( + config, collection, - optInStatus, - statsCollectionConfig + collectionEsClient, + this.usageCollection ); - if (optInStats && optInStats.length) { - this.logger.debug(`Got Opt In stats using ${collection.title} collection.`); - if (config.unencrypted) { - return optInStats; + + try { + const optInStats = await this.getOptInStatsForCollection( + collection, + optInStatus, + statsCollectionConfig + ); + if (optInStats && optInStats.length) { + this.logger.debug(`Got Opt In stats using ${collection.title} collection.`); + if (config.unencrypted) { + return optInStats; + } + return encryptTelemetry(optInStats, { useProdKey: this.isDistributable }); } - return encryptTelemetry(optInStats, { useProdKey: this.isDistributable }); + } catch (err) { + this.logger.debug(`Failed to collect any opt in stats with registered collections.`); + // swallow error to try next collection; } - } catch (err) { - this.logger.debug(`Failed to collect any opt in stats with registered collections.`); - // swallow error to try next collection; } } @@ -192,28 +208,32 @@ export class TelemetryCollectionManagerPlugin return []; } for (const collection of this.collections) { - const statsCollectionConfig = this.getStatsCollectionConfig( - config, - collection, - this.usageCollection - ); - try { - const usageData = await this.getUsageForCollection(collection, statsCollectionConfig); - if (usageData.length) { - this.logger.debug(`Got Usage using ${collection.title} collection.`); - if (config.unencrypted) { - return usageData; - } + const collectionEsClient = collection.esClientGetter(); + if (collectionEsClient !== undefined) { + const statsCollectionConfig = this.getStatsCollectionConfig( + config, + collection, + collectionEsClient, + this.usageCollection + ); + try { + const usageData = await this.getUsageForCollection(collection, statsCollectionConfig); + if (usageData.length) { + this.logger.debug(`Got Usage using ${collection.title} collection.`); + if (config.unencrypted) { + return usageData; + } - return encryptTelemetry(usageData.filter(isClusterOptedIn), { - useProdKey: this.isDistributable, - }); + return encryptTelemetry(usageData.filter(isClusterOptedIn), { + useProdKey: this.isDistributable, + }); + } + } catch (err) { + this.logger.debug( + `Failed to collect any usage with registered collection ${collection.title}.` + ); + // swallow error to try next collection; } - } catch (err) { - this.logger.debug( - `Failed to collect any usage with registered collection ${collection.title}.` - ); - // swallow error to try next collection; } } diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index 16f96c07fd8ea..44970df30fd16 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -17,8 +17,15 @@ * under the License. */ -import { LegacyAPICaller, Logger, KibanaRequest, ILegacyClusterClient } from 'kibana/server'; +import { + LegacyAPICaller, + Logger, + KibanaRequest, + ILegacyClusterClient, + IClusterClient, +} from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { ElasticsearchClient } from '../../../../src/core/server'; import { TelemetryCollectionManagerPlugin } from './plugin'; export interface TelemetryCollectionManagerPluginSetup { @@ -67,6 +74,7 @@ export interface StatsCollectionConfig { callCluster: LegacyAPICaller; start: string | number; end: string | number; + esClient: ElasticsearchClient; } export interface BasicStatsPayload { @@ -100,7 +108,7 @@ export interface ESLicense { } export interface StatsCollectionContext { - logger: Logger; + logger: Logger | Console; version: string; } @@ -130,6 +138,7 @@ export interface CollectionConfig< title: string; priority: number; esCluster: ILegacyClusterClient; + esClientGetter: () => IClusterClient | undefined; // --> by now we know that the client getter will return the IClusterClient but we assure that through a code check statsGetter: StatsGetter; clusterDetailsGetter: ClusterDetailsGetter; licenseGetter: LicenseGetter; @@ -145,5 +154,6 @@ export interface Collection< licenseGetter: LicenseGetter; clusterDetailsGetter: ClusterDetailsGetter; esCluster: ILegacyClusterClient; + esClientGetter: () => IClusterClient | undefined; // the collection could still return undefined for the es client getter. title: string; } diff --git a/src/plugins/usage_collection/README.md b/src/plugins/usage_collection/README.md index 0b1cca07de007..d8edc5bb8d18a 100644 --- a/src/plugins/usage_collection/README.md +++ b/src/plugins/usage_collection/README.md @@ -63,7 +63,7 @@ All you need to provide is a `type` for organizing your fields, `schema` field t total: 'long', }, }, - fetch: async (callCluster: APICluster) => { + fetch: async (callCluster: APICluster, esClient: IClusterClient) => { // query ES and get some data // summarize the data into a model @@ -86,9 +86,9 @@ Some background: - `MY_USAGE_TYPE` can be any string. It usually matches the plugin name. As a safety mechanism, we double check there are no duplicates at the moment of registering the collector. - The `fetch` method needs to support multiple contexts in which it is called. For example, when stats are pulled from a Kibana Metricbeat module, the Beat calls Kibana's stats API to invoke usage collection. -In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest`, where the request headers are expected to have read privilege on the entire `.kibana' index. +In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest` or `esClient` wraps `asCurrentUser`, where the request headers are expected to have read privilege on the entire `.kibana' index. -Note: there will be many cases where you won't need to use the `callCluster` function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS, or use other clients like a custom SavedObjects client. In that case it's up to the plugin to initialize those clients like the example below: +Note: there will be many cases where you won't need to use the `callCluster` (or `esClient`) function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS, or use other clients like a custom SavedObjects client. In that case it's up to the plugin to initialize those clients like the example below: ```ts // server/plugin.ts @@ -302,4 +302,4 @@ These saved objects are automatically consumed by the stats API and surfaced und By storing these metrics and their counts as key-value pairs, we can add more metrics without having to worry about exceeding the 1000-field soft limit in Elasticsearch. -The only caveat is that it makes it harder to consume in Kibana when analysing each entry in the array separately. In the telemetry team we are working to find a solution to this. We are building a new way of reporting telemetry called [Pulse](../../../rfcs/text/0008_pulse.md) that will help on making these UI-Metrics easier to consume. +The only caveat is that it makes it harder to consume in Kibana when analysing each entry in the array separately. In the telemetry team we are working to find a solution to this. diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index d57700024c088..365e1ce201337 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Logger, LegacyAPICaller } from 'kibana/server'; +import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; export type CollectorFormatForBulkUpload = (result: T) => { type: string; payload: U }; @@ -48,7 +48,7 @@ export interface CollectorOptions { type: string; init?: Function; schema?: MakeSchemaFrom>; // Using Required to enforce all optional keys in the object - fetch: (callCluster: LegacyAPICaller) => Promise | T; + fetch: (callCluster: LegacyAPICaller, esClient?: ElasticsearchClient) => Promise | T; /* * A hook for allowing the fetched data payload to be organized into a typed * data model for internal bulk upload. See defaultFormatterForBulkUpload for diff --git a/src/plugins/usage_collection/server/collector/collector_set.test.ts b/src/plugins/usage_collection/server/collector/collector_set.test.ts index 545642c5dcfa3..3f943ad8bf2ff 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.test.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.test.ts @@ -21,7 +21,7 @@ import { noop } from 'lodash'; import { Collector } from './collector'; import { CollectorSet } from './collector_set'; import { UsageCollector } from './usage_collector'; -import { loggingSystemMock } from '../../../../core/server/mocks'; +import { elasticsearchServiceMock, loggingSystemMock } from '../../../../core/server/mocks'; const logger = loggingSystemMock.createLogger(); @@ -42,6 +42,7 @@ describe('CollectorSet', () => { }); const mockCallCluster = jest.fn().mockResolvedValue({ passTest: 1000 }); + const mockEsClient = elasticsearchServiceMock.createClusterClient().asInternalUser; it('should throw an error if non-Collector type of object is registered', () => { const collectors = new CollectorSet({ logger }); @@ -85,7 +86,7 @@ describe('CollectorSet', () => { }) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(loggerSpies.debug).toHaveBeenCalledTimes(1); expect(loggerSpies.debug).toHaveBeenCalledWith( 'Fetching data from MY_TEST_COLLECTOR collector' @@ -110,7 +111,7 @@ describe('CollectorSet', () => { let result; try { - result = await collectors.bulkFetch(mockCallCluster); + result = await collectors.bulkFetch(mockCallCluster, mockEsClient); } catch (err) { // Do nothing } @@ -128,7 +129,7 @@ describe('CollectorSet', () => { }) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(result).toStrictEqual([ { type: 'MY_TEST_COLLECTOR', @@ -146,7 +147,7 @@ describe('CollectorSet', () => { } as any) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(result).toStrictEqual([ { type: 'MY_TEST_COLLECTOR', @@ -169,7 +170,7 @@ describe('CollectorSet', () => { }) ); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); expect(result).toStrictEqual([ { type: 'MY_TEST_COLLECTOR', diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index fce17a46b7168..6861be7f4f76b 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -18,7 +18,7 @@ */ import { snakeCase } from 'lodash'; -import { Logger, LegacyAPICaller } from 'kibana/server'; +import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import { Collector, CollectorOptions } from './collector'; import { UsageCollector } from './usage_collector'; @@ -117,8 +117,12 @@ export class CollectorSet { return allReady; }; + // all collections eventually pass through bulkFetch. + // the shape of the response is different when using the new ES client as is the error handling. + // We'll handle the refactor for using the new client in a follow up PR. public bulkFetch = async ( callCluster: LegacyAPICaller, + esClient: ElasticsearchClient, collectors: Map> = this.collectors ) => { const responses = await Promise.all( @@ -127,7 +131,7 @@ export class CollectorSet { try { return { type: collector.type, - result: await collector.fetch(callCluster), + result: await collector.fetch(callCluster, esClient), // each collector must ensure they handle the response appropriately. }; } catch (err) { this.logger.warn(err); @@ -149,9 +153,9 @@ export class CollectorSet { return this.makeCollectorSetFromArray(filtered); }; - public bulkFetchUsage = async (callCluster: LegacyAPICaller) => { + public bulkFetchUsage = async (callCluster: LegacyAPICaller, esClient: ElasticsearchClient) => { const usageCollectors = this.getFilteredCollectorSet((c) => c instanceof UsageCollector); - return await this.bulkFetch(callCluster, usageCollectors.collectors); + return await this.bulkFetch(callCluster, esClient, usageCollectors.collectors); }; // convert an array of fetched stats results into key/object diff --git a/src/plugins/usage_collection/server/routes/stats.ts b/src/plugins/usage_collection/server/routes/stats.ts index 7c64c9f180319..ef5da2eb11ba6 100644 --- a/src/plugins/usage_collection/server/routes/stats.ts +++ b/src/plugins/usage_collection/server/routes/stats.ts @@ -24,6 +24,7 @@ import { Observable } from 'rxjs'; import { first } from 'rxjs/operators'; import { + ElasticsearchClient, IRouter, LegacyAPICaller, MetricsServiceSetup, @@ -61,8 +62,11 @@ export function registerStatsRoute({ metrics: MetricsServiceSetup; overallStatus$: Observable; }) { - const getUsage = async (callCluster: LegacyAPICaller): Promise => { - const usage = await collectorSet.bulkFetchUsage(callCluster); + const getUsage = async ( + callCluster: LegacyAPICaller, + esClient: ElasticsearchClient + ): Promise => { + const usage = await collectorSet.bulkFetchUsage(callCluster, esClient); return collectorSet.toObject(usage); }; @@ -96,13 +100,14 @@ export function registerStatsRoute({ let extended; if (isExtended) { const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const collectorsReady = await collectorSet.areAllCollectorsReady(); if (shouldGetUsage && !collectorsReady) { return res.customError({ statusCode: 503, body: { message: STATS_NOT_READY_MESSAGE } }); } - const usagePromise = shouldGetUsage ? getUsage(callCluster) : Promise.resolve({}); + const usagePromise = shouldGetUsage ? getUsage(callCluster, esClient) : Promise.resolve({}); const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(callCluster)]); let modifiedUsage = usage; diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index 294f52cc3678f..8d3248ddf4369 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -20,6 +20,7 @@ import { CoreStart, CustomHttpResponseOptions, ResponseError, + IClusterClient, } from 'kibana/server'; import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; import { @@ -73,6 +74,7 @@ export class Plugin { private monitoringCore = {} as MonitoringCore; private legacyShimDependencies = {} as LegacyShimDependencies; private bulkUploader: IBulkUploader = {} as IBulkUploader; + private telemetryElasticsearchClient: IClusterClient | undefined; constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; @@ -143,9 +145,14 @@ export class Plugin { // Initialize telemetry if (plugins.telemetryCollectionManager) { - registerMonitoringCollection(plugins.telemetryCollectionManager, this.cluster, { - maxBucketSize: config.ui.max_bucket_size, - }); + registerMonitoringCollection( + plugins.telemetryCollectionManager, + this.cluster, + () => this.telemetryElasticsearchClient, + { + maxBucketSize: config.ui.max_bucket_size, + } + ); } // Register collector objects for stats to show up in the APIs @@ -229,7 +236,13 @@ export class Plugin { }; } - start() {} + start({ elasticsearch }: CoreStart) { + // TODO: For the telemetry plugin to work, we need to provide the new ES client. + // The new client should be inititalized with a similar config to `this.cluster` but, since we're not using + // the new client in Monitoring Telemetry collection yet, setting the local client allos progress for now. + // We will update the client in a follow up PR. + this.telemetryElasticsearchClient = elasticsearch.client; + } stop() { if (this.cluster) { diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts index f0ad6399c6c72..89f09d349014f 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts @@ -15,6 +15,7 @@ describe('get_all_stats', () => { const start = 0; const end = 1; const callCluster = sinon.stub(); + const esClient = sinon.stub(); const esClusters = [ { cluster_uuid: 'a' }, @@ -176,6 +177,7 @@ describe('get_all_stats', () => { [{ clusterUuid: 'a' }], { callCluster: callCluster as any, + esClient: esClient as any, usageCollection: {} as any, start, end, @@ -201,6 +203,7 @@ describe('get_all_stats', () => { [], { callCluster: callCluster as any, + esClient: esClient as any, usageCollection: {} as any, start, end, diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts index 726db1706758d..1170380b26ac8 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts @@ -21,7 +21,6 @@ type PromiseReturnType any> = ReturnType extend export interface CustomContext { maxBucketSize: number; } - /** * Get statistics for all products joined by Elasticsearch cluster. * Returns the array of clusters joined with the Kibana and Logstash instances. @@ -29,7 +28,7 @@ export interface CustomContext { */ export const getAllStats: StatsGetter = async ( clustersDetails, - { callCluster, start, end }, + { callCluster, start, end, esClient }, { maxBucketSize } ) => { const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts index 519dcc38875f5..b2f3cb6c61526 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts @@ -5,6 +5,7 @@ */ import sinon from 'sinon'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { getClusterUuids, fetchClusterUuids, @@ -13,6 +14,7 @@ import { describe('get_cluster_uuids', () => { const callCluster = sinon.stub(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const response = { aggregations: { cluster_uuids: { @@ -30,7 +32,7 @@ describe('get_cluster_uuids', () => { it('returns cluster UUIDs', async () => { callCluster.withArgs('search').returns(Promise.resolve(response)); expect( - await getClusterUuids({ callCluster, start, end, usageCollection: {} as any }, { + await getClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, { maxBucketSize: 1, } as any) ).toStrictEqual(expectedUuids); @@ -41,7 +43,7 @@ describe('get_cluster_uuids', () => { it('searches for clusters', async () => { callCluster.returns(Promise.resolve(response)); expect( - await fetchClusterUuids({ callCluster, start, end, usageCollection: {} as any }, { + await fetchClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, { maxBucketSize: 1, } as any) ).toStrictEqual(response); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts index 1e5549fe4d800..3648ae4bd8551 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ILegacyCustomClusterClient } from 'kibana/server'; +import { ILegacyCustomClusterClient, IClusterClient } from 'kibana/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { getAllStats, CustomContext } from './get_all_stats'; import { getClusterUuids } from './get_cluster_uuids'; @@ -13,10 +13,12 @@ import { getLicenses } from './get_licenses'; export function registerMonitoringCollection( telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, esCluster: ILegacyCustomClusterClient, + esClientGetter: () => IClusterClient | undefined, customContext: CustomContext ) { telemetryCollectionManager.setCollection({ esCluster, + esClientGetter, title: 'monitoring', priority: 2, statsGetter: getAllStats, diff --git a/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts b/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts index 3f01d7423eded..6ef44e325b0a7 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts @@ -4,7 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { PluginInitializerContext, CoreSetup, CoreStart, Plugin } from 'kibana/server'; +import { + PluginInitializerContext, + CoreSetup, + CoreStart, + Plugin, + IClusterClient, +} from 'kibana/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { getClusterUuids, getLocalLicense } from '../../../../src/plugins/telemetry/server'; import { getStatsWithXpack } from './telemetry_collection'; @@ -14,11 +20,13 @@ interface TelemetryCollectionXpackDepsSetup { } export class TelemetryCollectionXpackPlugin implements Plugin { + private elasticsearchClient?: IClusterClient; constructor(initializerContext: PluginInitializerContext) {} public setup(core: CoreSetup, { telemetryCollectionManager }: TelemetryCollectionXpackDepsSetup) { telemetryCollectionManager.setCollection({ esCluster: core.elasticsearch.legacy.client, + esClientGetter: () => this.elasticsearchClient, title: 'local_xpack', priority: 1, statsGetter: getStatsWithXpack, @@ -27,5 +35,7 @@ export class TelemetryCollectionXpackPlugin implements Plugin { }); } - public start(core: CoreStart) {} + public start(core: CoreStart) { + this.elasticsearchClient = core.elasticsearch.client; + } } diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js deleted file mode 100644 index eb03701fd195b..0000000000000 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; - -import { TIMEOUT } from '../constants'; -import { getXPackUsage } from '../get_xpack'; - -function mockGetXPackUsage(callCluster, usage, req) { - callCluster - .withArgs(req, 'transport.request', { - method: 'GET', - path: '/_xpack/usage', - query: { - master_timeout: TIMEOUT, - }, - }) - .returns(usage); - - callCluster - .withArgs('transport.request', { - method: 'GET', - path: '/_xpack/usage', - query: { - master_timeout: TIMEOUT, - }, - }) - .returns(usage); -} - -describe('get_xpack', () => { - describe('getXPackUsage', () => { - it('uses callCluster to get /_xpack/usage API', () => { - const response = Promise.resolve({}); - const callCluster = sinon.stub(); - - mockGetXPackUsage(callCluster, response); - - expect(getXPackUsage(callCluster)).to.be(response); - }); - }); -}); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts index 24382fb89d337..a4806cefeef3d 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { coreMock } from '../../../../../src/core/server/mocks'; +import { coreMock, elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { getStatsWithXpack } from './get_stats_with_xpack'; const kibana = { @@ -55,32 +55,34 @@ const mockUsageCollection = (kibanaUsage = kibana) => ({ describe('Telemetry Collection: Get Aggregated Stats', () => { test('OSS-like telemetry (no license nor X-Pack telemetry)', async () => { - const callCluster = jest.fn(async (method: string, options: { path?: string }) => { - switch (method) { - case 'transport.request': - if (options.path === '/_license' || options.path === '/_xpack/usage') { - // eslint-disable-next-line no-throw-literal - throw { statusCode: 404 }; - } else if (options.path === '/_nodes/usage') { - return { - cluster_name: 'test cluster', - nodes: nodesUsage, - }; - } - return {}; - case 'info': - return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } }; - default: - return {}; + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // mock for xpack.usage should throw a 404 for this test + esClient.xpack.usage.mockRejectedValue(new Error('Not Found')); + // mock for license should throw a 404 for this test + esClient.license.get.mockRejectedValue(new Error('Not Found')); + // mock for nodes usage should resolve for this test + esClient.nodes.usage.mockResolvedValue( + // @ts-ignore we only care about the response body + { body: { cluster_name: 'test cluster', nodes: nodesUsage } } + ); + // mock for info should resolve for this test + esClient.info.mockResolvedValue( + // @ts-ignore we only care about the response body + { + body: { + cluster_uuid: 'test', + cluster_name: 'test', + version: { number: '8.0.0' }, + }, } - }); + ); const usageCollection = mockUsageCollection(); const context = getContext(); const stats = await getStatsWithXpack( [{ clusterUuid: '1234' }], { - callCluster, + esClient, usageCollection, } as any, context @@ -93,36 +95,39 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { }); test('X-Pack telemetry (license + X-Pack)', async () => { - const callCluster = jest.fn(async (method: string, options: { path?: string }) => { - switch (method) { - case 'transport.request': - if (options.path === '/_license') { - return { - license: { type: 'basic' }, - }; - } - if (options.path === '/_xpack/usage') { - return {}; - } - if (options.path === '/_nodes/usage') { - return { - cluster_name: 'test cluster', - nodes: nodesUsage, - }; - } - case 'info': - return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } }; - default: - return {}; + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // mock for license should return a basic license + esClient.license.get.mockResolvedValue( + // @ts-ignore we only care about the response body + { body: { license: { type: 'basic' } } } + ); + // mock for xpack usage should return an empty object + esClient.xpack.usage.mockResolvedValue( + // @ts-ignore we only care about the response body + { body: {} } + ); + // mock for nodes usage should return the cluster name and nodes usage + esClient.nodes.usage.mockResolvedValue( + // @ts-ignore we only care about the response body + { body: { cluster_name: 'test cluster', nodes: nodesUsage } } + ); + esClient.info.mockResolvedValue( + // @ts-ignore we only care about the response body + { + body: { + cluster_uuid: 'test', + cluster_name: 'test', + version: { number: '8.0.0' }, + }, } - }); + ); const usageCollection = mockUsageCollection(); const context = getContext(); const stats = await getStatsWithXpack( [{ clusterUuid: '1234' }], { - callCluster, + esClient, usageCollection, } as any, context diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts index 3fcd25c31e71e..87e3d0a9613da 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts @@ -17,9 +17,9 @@ export const getStatsWithXpack: StatsGetter<{}, TelemetryAggregatedStats> = asyn config, context ) { - const { callCluster } = config; + const { esClient } = config; const clustersLocalStats = await getLocalStats(clustersDetails, config, context); - const xpack = await getXPackUsage(callCluster).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails. + const xpack = await getXPackUsage(esClient).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails. return clustersLocalStats.map((localStats) => { if (xpack) { diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts new file mode 100644 index 0000000000000..106df71e46c7e --- /dev/null +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { getXPackUsage } from './get_xpack'; + +describe('get_xpack', () => { + describe('getXPackUsage', () => { + it('uses esClient to get /_xpack/usage API', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // @ts-ignore we only care about the response body + esClient.xpack.usage.mockResolvedValue({ body: {} }); + const result = await getXPackUsage(esClient); + expect(result).toEqual({}); + }); + }); +}); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts index 0ff75717c8b65..4dbf2052be28c 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'src/core/server'; import { TIMEOUT } from './constants'; /** @@ -14,12 +14,7 @@ import { TIMEOUT } from './constants'; * * Like any X-Pack related API, X-Pack must installed for this to work. */ -export function getXPackUsage(callCluster: LegacyAPICaller) { - return callCluster('transport.request', { - method: 'GET', - path: '/_xpack/usage', - query: { - master_timeout: TIMEOUT, - }, - }); +export async function getXPackUsage(esClient: ElasticsearchClient) { + const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT }); + return body; }