[Telemetry] collector set to np (#51618)

* first iteration

* local collection ready

* type check

* fix collectorSet tests

* unskip test

* ordering

* collectors as array in constructor

* update README files

* update README and canvas to check for optional dep

* update README with more details

* Add file path for README example

* type UsageCollectionSetup

* run type check after refactor

.github/CODEOWNERS

@@ -77,6 +77,7 @@
 /src/dev/i18n @elastic/kibana-stack-services
 /packages/kbn-analytics/ @elastic/kibana-stack-services
 /src/legacy/core_plugins/ui_metric/ @elastic/kibana-stack-services
+/src/plugins/usage_collection/ @elastic/kibana-stack-services
 /x-pack/legacy/plugins/telemetry @elastic/kibana-stack-services
 /x-pack/legacy/plugins/alerting @elastic/kibana-stack-services
 /x-pack/legacy/plugins/actions @elastic/kibana-stack-services

src/core/CONVENTIONS.md

@@ -210,3 +210,40 @@ export class Plugin {
   }
 }
 ```
+
+### Usage Collection
+
+For creating and registering a Usage Collector. Collectors would be defined in a separate directory `server/collectors/register.ts`. You can read more about usage collectors on `src/plugins/usage_collection/README.md`.
+
+```ts
+// server/collectors/register.ts
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
+
+export function registerMyPluginUsageCollector(usageCollection?: UsageCollectionSetup): void {
+  // usageCollection is an optional dependency, so make sure to return if it is not registered.
+  if (!usageCollection) {
+    return;
+  }
+
+  // create usage collector
+  const myCollector = usageCollection.makeUsageCollector({
+    type: MY_USAGE_TYPE,
+    fetch: async (callCluster: CallCluster) => {
+
+    // query ES and get some data
+    // summarize the data into a model
+    // return the modeled object that includes whatever you want to track
+
+      return {
+        my_objects: {
+          total: SOME_NUMBER
+        }
+      };
+    },
+  });
+
+  // register usage collector
+  usageCollection.registerCollector(myCollector);
+}
+```

src/legacy/core_plugins/kibana/index.js

@@ -324,6 +324,7 @@ export default function (kibana) {
     },
 
     init: async function (server) {
+      const { usageCollection } = server.newPlatform.setup.plugins;
       // uuid
       await manageUuid(server);
       // routes
@@ -338,8 +339,8 @@ export default function (kibana) {
       registerKqlTelemetryApi(server);
       registerFieldFormats(server);
       registerTutorials(server);
-      makeKQLUsageCollector(server);
-      registerCspCollector(server);
+      makeKQLUsageCollector(usageCollection, server);
+      registerCspCollector(usageCollection, server);
       server.expose('systemApi', systemApi);
       server.injectUiAppVars('kibana', () => injectVars(server));
     },

src/legacy/core_plugins/kibana/server/lib/csp_usage_collector/csp_collector.ts

@@ -19,6 +19,7 @@
 
 import { Server } from 'hapi';
 import { createCSPRuleString, DEFAULT_CSP_RULES } from '../../../../../server/csp';
+import { UsageCollectionSetup } from '../../../../../../plugins/usage_collection/server';
 
 export function createCspCollector(server: Server) {
   return {
@@ -42,8 +43,7 @@ export function createCspCollector(server: Server) {
   };
 }
 
-export function registerCspCollector(server: Server): void {
-  const { collectorSet } = server.usage;
-  const collector = collectorSet.makeUsageCollector(createCspCollector(server));
-  collectorSet.register(collector);
+export function registerCspCollector(usageCollection: UsageCollectionSetup, server: Server): void {
+  const collector = usageCollection.makeUsageCollector(createCspCollector(server));
+  usageCollection.registerCollector(collector);
 }

src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.js

@@ -19,14 +19,14 @@
 
 import { fetchProvider } from './fetch';
 
-export function makeKQLUsageCollector(server) {
+export function makeKQLUsageCollector(usageCollection, server) {
   const index = server.config().get('kibana.index');
   const fetch = fetchProvider(index);
-  const kqlUsageCollector = server.usage.collectorSet.makeUsageCollector({
+  const kqlUsageCollector = usageCollection.makeUsageCollector({
     type: 'kql',
     fetch,
     isReady: () => true,
   });
 
-  server.usage.collectorSet.register(kqlUsageCollector);
+  usageCollection.registerCollector(kqlUsageCollector);
 }

src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.test.js

@@ -20,29 +20,30 @@
 import { makeKQLUsageCollector } from './make_kql_usage_collector';
 
 describe('makeKQLUsageCollector', () => {
-
   let server;
   let makeUsageCollectorStub;
   let registerStub;
+  let usageCollection;
 
   beforeEach(() => {
     makeUsageCollectorStub = jest.fn();
     registerStub = jest.fn();
+    usageCollection = {
+      makeUsageCollector: makeUsageCollectorStub,
+      registerCollector: registerStub,
+    };
     server = {
-      usage: {
-        collectorSet: { makeUsageCollector: makeUsageCollectorStub, register: registerStub },
-      },
       config: () => ({ get: () => '.kibana' })
     };
   });
 
-  it('should call collectorSet.register', () => {
-    makeKQLUsageCollector(server);
+  it('should call registerCollector', () => {
+    makeKQLUsageCollector(usageCollection, server);
     expect(registerStub).toHaveBeenCalledTimes(1);
   });
 
   it('should call makeUsageCollector with type = kql', () => {
-    makeKQLUsageCollector(server);
+    makeKQLUsageCollector(usageCollection, server);
     expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1);
     expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('kql');
   });

src/legacy/core_plugins/telemetry/README.md

@@ -0,0 +1,9 @@
+# Kibana Telemetry Service
+
+Telemetry allows Kibana features to have usage tracked in the wild. The general term "telemetry" refers to multiple things:
+
+1. Integrating with the telemetry service to express how to collect usage data (Collecting).
+2. Sending a payload of usage data up to Elastic's telemetry cluster.
+3. Viewing usage data in the Kibana instance of the telemetry cluster (Viewing).
+
+This plugin is responsible for sending usage data to the telemetry cluster. For collecting usage data, use 

src/legacy/core_plugins/telemetry/index.ts

@@ -27,14 +27,7 @@ import { i18n } from '@kbn/i18n';
 import mappings from './mappings.json';
 import { CONFIG_TELEMETRY, getConfigTelemetryDesc } from './common/constants';
 import { getXpackConfigWithDeprecated } from './common/get_xpack_config_with_deprecated';
-import { telemetryPlugin, replaceTelemetryInjectedVars, FetcherTask } from './server';
-
-import {
-  createLocalizationUsageCollector,
-  createTelemetryUsageCollector,
-  createUiMetricUsageCollector,
-  createTelemetryPluginUsageCollector,
-} from './server/collectors';
+import { telemetryPlugin, replaceTelemetryInjectedVars, FetcherTask, PluginsSetup } from './server';
 
 const ENDPOINT_VERSION = 'v2';
 
@@ -123,6 +116,7 @@ const telemetry = (kibana: any) => {
       fetcherTask.start();
     },
     init(server: Server) {
+      const { usageCollection } = server.newPlatform.setup.plugins;
       const initializerContext = {
         env: {
           packageInfo: {
@@ -149,12 +143,11 @@ const telemetry = (kibana: any) => {
         log: server.log,
       } as any) as CoreSetup;
 
-      telemetryPlugin(initializerContext).setup(coreSetup);
-      // register collectors
-      server.usage.collectorSet.register(createTelemetryPluginUsageCollector(server));
-      server.usage.collectorSet.register(createLocalizationUsageCollector(server));
-      server.usage.collectorSet.register(createTelemetryUsageCollector(server));
-      server.usage.collectorSet.register(createUiMetricUsageCollector(server));
+      const pluginsSetup: PluginsSetup = {
+        usageCollection,
+      };
+
+      telemetryPlugin(initializerContext).setup(coreSetup, pluginsSetup, server);
     },
   });
 };

src/legacy/core_plugins/telemetry/server/collection_manager.ts

@@ -19,6 +19,7 @@
 
 import { encryptTelemetry } from './collectors';
 import { CallCluster } from '../../elasticsearch';
+import { UsageCollectionSetup } from '../../../../plugins/usage_collection/server';
 
 export type EncryptedStatsGetterConfig = { unencrypted: false } & {
   server: any;
@@ -37,6 +38,7 @@ export interface ClusterDetails {
 }
 
 export interface StatsCollectionConfig {
+  usageCollection: UsageCollectionSetup;
   callCluster: CallCluster;
   server: any;
   start: string;
@@ -112,7 +114,8 @@ export class TelemetryCollectionManager {
       ? (...args: any[]) => callWithRequest(config.req, ...args)
       : callWithInternalUser;
 
-    return { server, callCluster, start, end };
+    const { usageCollection } = server.newPlatform.setup.plugins;
+    return { server, callCluster, start, end, usageCollection };
   };
 
   private getOptInStatsForCollection = async (

src/legacy/core_plugins/telemetry/server/collectors/index.ts

@@ -18,7 +18,7 @@
  */
 
 export { encryptTelemetry } from './encryption';
-export { createTelemetryUsageCollector } from './usage';
-export { createUiMetricUsageCollector } from './ui_metric';
-export { createLocalizationUsageCollector } from './localization';
-export { createTelemetryPluginUsageCollector } from './telemetry_plugin';
+export { registerTelemetryUsageCollector } from './usage';
+export { registerUiMetricUsageCollector } from './ui_metric';
+export { registerLocalizationUsageCollector } from './localization';
+export { registerTelemetryPluginUsageCollector } from './telemetry_plugin';

src/legacy/core_plugins/telemetry/server/collectors/localization/index.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export { createLocalizationUsageCollector } from './telemetry_localization_collector';
+export { registerLocalizationUsageCollector } from './telemetry_localization_collector';

src/legacy/core_plugins/telemetry/server/collectors/localization/telemetry_localization_collector.ts

@@ -21,6 +21,7 @@ import { i18nLoader } from '@kbn/i18n';
 import { size } from 'lodash';
 import { getIntegrityHashes, Integrities } from './file_integrity';
 import { KIBANA_LOCALIZATION_STATS_TYPE } from '../../../common/constants';
+import { UsageCollectionSetup } from '../../../../../../plugins/usage_collection/server';
 export interface UsageStats {
   locale: string;
   integrities: Integrities;
@@ -51,15 +52,15 @@ export function createCollectorFetch(server: any) {
   };
 }
 
-/*
- * @param {Object} server
- * @return {Object} kibana usage stats type collection object
- */
-export function createLocalizationUsageCollector(server: any) {
-  const { collectorSet } = server.usage;
-  return collectorSet.makeUsageCollector({
+export function registerLocalizationUsageCollector(
+  usageCollection: UsageCollectionSetup,
+  server: any
+) {
+  const collector = usageCollection.makeUsageCollector({
     type: KIBANA_LOCALIZATION_STATS_TYPE,
     isReady: () => true,
     fetch: createCollectorFetch(server),
   });
+
+  usageCollection.registerCollector(collector);
 }

src/legacy/core_plugins/telemetry/server/collectors/telemetry_plugin/index.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export { createTelemetryPluginUsageCollector } from './telemetry_plugin_collector';
+export { registerTelemetryPluginUsageCollector } from './telemetry_plugin_collector';

src/legacy/core_plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts

@@ -20,6 +20,8 @@
 import { TELEMETRY_STATS_TYPE } from '../../../common/constants';
 import { getTelemetrySavedObject, TelemetrySavedObject } from '../../telemetry_repository';
 import { getTelemetryOptIn, getTelemetrySendUsageFrom } from '../../telemetry_config';
+import { UsageCollectionSetup } from '../../../../../../plugins/usage_collection/server';
+
 export interface TelemetryUsageStats {
   opt_in_status?: boolean | null;
   usage_fetcher?: 'browser' | 'server';
@@ -61,15 +63,15 @@ export function createCollectorFetch(server: any) {
   };
 }
 
-/*
- * @param {Object} server
- * @return {Object} kibana usage stats type collection object
- */
-export function createTelemetryPluginUsageCollector(server: any) {
-  const { collectorSet } = server.usage;
-  return collectorSet.makeUsageCollector({
+export function registerTelemetryPluginUsageCollector(
+  usageCollection: UsageCollectionSetup,
+  server: any
+) {
+  const collector = usageCollection.makeUsageCollector({
     type: TELEMETRY_STATS_TYPE,
     isReady: () => true,
     fetch: createCollectorFetch(server),
   });
+
+  usageCollection.registerCollector(collector);
 }

src/legacy/core_plugins/telemetry/server/collectors/ui_metric/index.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export { createUiMetricUsageCollector } from './telemetry_ui_metric_collector';
+export { registerUiMetricUsageCollector } from './telemetry_ui_metric_collector';

src/legacy/core_plugins/telemetry/server/collectors/ui_metric/telemetry_ui_metric_collector.ts

@@ -18,10 +18,10 @@
  */
 
 import { UI_METRIC_USAGE_TYPE } from '../../../common/constants';
+import { UsageCollectionSetup } from '../../../../../../plugins/usage_collection/server';
 
-export function createUiMetricUsageCollector(server: any) {
-  const { collectorSet } = server.usage;
-  return collectorSet.makeUsageCollector({
+export function registerUiMetricUsageCollector(usageCollection: UsageCollectionSetup, server: any) {
+  const collector = usageCollection.makeUsageCollector({
     type: UI_METRIC_USAGE_TYPE,
     fetch: async () => {
       const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects;
@@ -55,4 +55,6 @@ export function createUiMetricUsageCollector(server: any) {
     },
     isReady: () => true,
   });
+
+  usageCollection.registerCollector(collector);
 }

src/legacy/core_plugins/telemetry/server/collectors/usage/index.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export { createTelemetryUsageCollector } from './telemetry_usage_collector';
+export { registerTelemetryUsageCollector } from './telemetry_usage_collector';

src/legacy/core_plugins/telemetry/server/collectors/usage/telemetry_usage_collector.test.ts

@@ -25,20 +25,15 @@ import {
   createTelemetryUsageCollector,
   isFileReadable,
   readTelemetryFile,
-  KibanaHapiServer,
   MAX_FILE_SIZE,
 } from './telemetry_usage_collector';
 
-const getMockServer = (): KibanaHapiServer =>
-  ({
-    usage: {
-      collectorSet: { makeUsageCollector: jest.fn().mockImplementationOnce((arg: object) => arg) },
-    },
-  } as KibanaHapiServer & Server);
+const mockUsageCollector = () => ({
+  makeUsageCollector: jest.fn().mockImplementationOnce((arg: object) => arg),
+});
 
-const serverWithConfig = (configPath: string): KibanaHapiServer & Server => {
+const serverWithConfig = (configPath: string): Server => {
   return {
-    ...getMockServer(),
     config: () => ({
       get: (key: string) => {
         if (key !== 'telemetry.config' && key !== 'xpack.xpack_main.telemetry.config') {
@@ -48,7 +43,7 @@ const serverWithConfig = (configPath: string): KibanaHapiServer & Server => {
         return configPath;
       },
     }),
-  } as KibanaHapiServer & Server;
+  } as Server;
 };
 
 describe('telemetry_usage_collector', () => {
@@ -130,14 +125,15 @@ describe('telemetry_usage_collector', () => {
   });
 
   describe('createTelemetryUsageCollector', () => {
-    test('calls `collectorSet.makeUsageCollector`', async () => {
+    test('calls `makeUsageCollector`', async () => {
       // note: it uses the file's path to get the directory, then looks for 'telemetry.yml'
       // exclusively, which is indirectly tested by passing it the wrong "file" in the same
       // dir
-      const server: KibanaHapiServer & Server = serverWithConfig(tempFiles.unreadable);
+      const server: Server = serverWithConfig(tempFiles.unreadable);
 
       // the `makeUsageCollector` is mocked above to return the argument passed to it
-      const collectorOptions = createTelemetryUsageCollector(server);
+      const usageCollector = mockUsageCollector() as any;
+      const collectorOptions = createTelemetryUsageCollector(usageCollector, server);
 
       expect(collectorOptions.type).toBe('static_telemetry');
       expect(await collectorOptions.fetch()).toEqual(expectedObject);