[Telemetry] Report data shippers

src/plugins/telemetry/server/index.ts

@@ -47,4 +47,8 @@ export {
   getLocalLicense,
   getLocalStats,
   TelemetryLocalStats,
+  DATA_TELEMETRY_ID,
+  DataTelemetryIndex,
+  DataTelemetryPayload,
+  buildDataTelemetryPayload,
 } from './telemetry_collection';

src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js

@@ -135,23 +135,24 @@ describe('get_local_stats', () => {
 
   describe('handleLocalStats', () => {
     it('returns expected object without xpack and kibana data', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStats, void 0, void 0, context);
       expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
       expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
       expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
       expect(result.version).to.be('2.3.4');
       expect(result.collection).to.be('local');
       expect(result.license).to.be(undefined);
-      expect(result.stack_stats).to.eql({ kibana: undefined });
+      expect(result.stack_stats).to.eql({ kibana: undefined, data: undefined });
     });
 
     it('returns expected object with xpack', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStats, void 0, void 0, context);
       const { stack_stats: stack, ...cluster } = result;
       expect(cluster.collection).to.be(combinedStatsResult.collection);
       expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
       expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name);
       expect(stack.kibana).to.be(undefined); // not mocked for this test
+      expect(stack.data).to.be(undefined); // not mocked for this test
 
       expect(cluster.version).to.eql(combinedStatsResult.version);
       expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats);

src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/constants.ts

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const DATA_TELEMETRY_ID = 'data';
+
+export const DATA_KNOWN_TYPES = ['logs', 'traces', 'metrics'] as const;
+
+export type DataTelemetryType = typeof DATA_KNOWN_TYPES[number];
+
+export type DataPatternName = typeof DATA_DATASETS_INDEX_PATTERNS[number]['patternName'];
+
+// TODO: Ideally this list should be updated from an external public URL (similar to the newsfeed)
+// But it's good to have a minimum list shipped with the build.
+export const DATA_DATASETS_INDEX_PATTERNS = [
+  // Security - Elastic
+  { pattern: 'auditbeat-*', patternName: 'auditbeat', shipper: 'auditbeat' },
+  { pattern: 'winlogbeat-*', patternName: 'winlogbeat', shipper: 'winlogbeat' },
+  { pattern: 'packetbeat-*', patternName: 'packetbeat', shipper: 'packetbeat' },
+  // Security - 3rd party
+  { pattern: '*tomcat*', patternName: 'tomcat' },
+  { pattern: '*apache*', patternName: 'apache' }, // Already in Observability (keeping it in here for documentation)
+  { pattern: '*artifactory*', patternName: 'artifactory' },
+  { pattern: '*aruba*', patternName: 'aruba' },
+  { pattern: '*barracuda*', patternName: 'barracuda' },
+  { pattern: '*cef*', patternName: 'cef' },
+  { pattern: '*checkpoint*', patternName: 'checkpoint' },
+  { pattern: '*cisco*', patternName: 'cisco' },
+  { pattern: '*citrix*', patternName: 'citrix' },
+  { pattern: '*cyberark*', patternName: 'cyberark' },
+  { pattern: '*cylance*', patternName: 'cylance' },
+  { pattern: '*fortinet*', patternName: 'fortinet' },
+  { pattern: '*infoblox*', patternName: 'infoblox' },
+  { pattern: '*kaspersky*', patternName: 'kaspersky' },
+  { pattern: '*mcafee*', patternName: 'mcafee' },
+  { pattern: '*paloaltonetworks*', patternName: 'paloaltonetworks' },
+  { pattern: '*pan*', patternName: 'paloaltonetworks' },
+  { pattern: '*rsa*', patternName: 'rsa' },
+  { pattern: '*snort*', patternName: 'snort' },
+  { pattern: '*sonicwall*', patternName: 'sonicwall' },
+  { pattern: '*sophos*', patternName: 'sophos' },
+  { pattern: '*squid*', patternName: 'squid' },
+  { pattern: '*symantec*', patternName: 'symantec' },
+  { pattern: '*tippingpoint*', patternName: 'tippingpoint' },
+  { pattern: '*trendmicro*', patternName: 'trendmicro' },
+  { pattern: '*tripwire*', patternName: 'tripwire' },
+  { pattern: '*zscaler*', patternName: 'zscaler' },
+  { pattern: '*zeek*', patternName: 'zeek' },
+  { pattern: '*sigma_doc*', patternName: 'sigma_doc' },
+  { pattern: '*bro*', patternName: 'bro' },
+  { pattern: '*suricata*', patternName: 'suricata' },
+  { pattern: '*fsf*', patternName: 'fsf' },
+  { pattern: '*wazuh*', patternName: 'wazuh' },
+
+  // Enterprise Search - Elastic
+  { pattern: '.ent-search-*', patternName: 'enterprise-search' },
+  { pattern: '.app-search-*', patternName: 'app-search' },
+  // Enterprise Search - 3rd party
+  { pattern: '*magento2*', patternName: 'magento2' },
+  { pattern: '*magento*', patternName: 'magento' },
+  { pattern: '*shopify*', patternName: 'shopify' },
+  { pattern: '*wordpress*', patternName: 'wordpress' },
+  { pattern: '*wp*', patternName: 'wordpress' },
+  { pattern: '*drupal*', patternName: 'drupal' },
+  { pattern: '*joomla*', patternName: 'joomla' },
+  { pattern: '*search*', patternName: 'search' },
+  { pattern: '*wix*', patternName: 'wix' },
+  { pattern: '*sharepoint*', patternName: 'sharepoint' },
+  { pattern: '*squarespace*', patternName: 'squarespace' },
+  { pattern: '*aem*', patternName: 'aem' },
+  { pattern: '*sitecore*', patternName: 'sitecore' },
+  { pattern: '*weebly*', patternName: 'weebly' },
+  { pattern: '*acquia*', patternName: 'acquia' },
+
+  // Observability - Elastic
+  { pattern: 'filebeat-*', patternName: 'filebeat', shipper: 'filebeat' },
+  { pattern: 'metricbeat-*', patternName: 'metricbeat', shipper: 'metricbeat' },
+  { pattern: 'apm-*', patternName: 'apm', shipper: 'apm' },
+  { pattern: 'functionbeat-*', patternName: 'functionbeat', shipper: 'functionbeat' },
+  { pattern: 'heartbeat-*', patternName: 'heartbeat', shipper: 'heartbeat' },
+  { pattern: 'logstash-*', patternName: 'logstash', shipper: 'logstash' },
+  // Observability - 3rd party
+  { pattern: 'fluentd*', patternName: 'fluentd' },
+  { pattern: 'telegraf*', patternName: 'telegraf' },
+  { pattern: 'prometheusbeat*', patternName: 'prometheusbeat' },
+  { pattern: 'fluentbit*', patternName: 'fluentbit' },
+  { pattern: '*nginx*', patternName: 'nginx' },
+  { pattern: '*apache*', patternName: 'apache' }, // Already in Security (keeping it in here for documentation)
+  { pattern: '*logs*', patternName: 'third-party-logs' },
+] as const;

src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts

@@ -0,0 +1,234 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { buildDataTelemetryPayload, getDataTelemetry } from './get_data_telemetry';
+
+describe('get_data_telemetry', () => {
+  describe('buildDataTelemetryPayload', () => {
+    test('return the base object when no indices provided', () => {
+      expect(buildDataTelemetryPayload([])).toStrictEqual([]);
+    });
+
+    test('return the base object when no matching indices provided', () => {
+      expect(
+        buildDataTelemetryPayload([
+          { name: 'no__way__this__can_match_anything', sizeInBytes: 10 },
+          { name: '.kibana-event-log-8.0.0' },
+        ])
+      ).toStrictEqual([]);
+    });
+
+    test('matches some indices and puts them in their own category', () => {
+      expect(
+        buildDataTelemetryPayload([
+          // APM Indices have known shipper (so we can infer the datasetType from mapping constant)
+          { name: 'apm-7.7.0-error-000001', shipper: 'apm', isECS: true },
+          { name: 'apm-7.7.0-metric-000001', shipper: 'apm', isECS: true },
+          { name: 'apm-7.7.0-onboarding-2020.05.17', shipper: 'apm', isECS: true },
+          { name: 'apm-7.7.0-profile-000001', shipper: 'apm', isECS: true },
+          { name: 'apm-7.7.0-span-000001', shipper: 'apm', isECS: true },
+          { name: 'apm-7.7.0-transaction-000001', shipper: 'apm', isECS: true },
+          // Packetbeat indices with known shipper (we can infer datasetType from mapping constant)
+          { name: 'packetbeat-7.7.0-2020.06.11-000001', shipper: 'packetbeat', isECS: true },
+          // Matching patterns from the list => known datasetName but the rest is unknown
+          { name: 'filebeat-12314', docCount: 100, sizeInBytes: 10 },
+          { name: 'metricbeat-1234', docCount: 100, sizeInBytes: 10, isECS: false },
+          { name: '.app-search-1234', docCount: 0 },
+          // Matching pattern with datasetName and datasetType set but shipper unknown
+          { name: 'my_logs_custom', docCount: 1000, sizeInBytes: 10 },
+          { name: 'my_logs', docCount: 100, sizeInBytes: 10, isECS: true },
+          { name: 'logs_custom' },
+          { name: 'logs-custom-index-1234' },
+          // New Indexing strategy: everything can be inferred from the constant_keyword values
+          {
+            name: 'logs-nginx.access-default-000001',
+            datasetName: 'nginx.access',
+            datasetType: 'logs',
+            shipper: 'filebeat',
+            isECS: true,
+            docCount: 1000,
+            sizeInBytes: 1000,
+          },
+          {
+            name: 'logs-nginx.access-default-000002',
+            datasetName: 'nginx.access',
+            datasetType: 'logs',
+            shipper: 'filebeat',
+            isECS: true,
+            docCount: 1000,
+            sizeInBytes: 60,
+          },
+        ])
+      ).toStrictEqual([
+        {
+          shipper: 'apm',
+          index_count: 6,
+          ecs_index_count: 6,
+        },
+        {
+          shipper: 'packetbeat',
+          index_count: 1,
+          ecs_index_count: 1,
+        },
+        {
+          pattern_name: 'filebeat',
+          shipper: 'filebeat',
+          index_count: 1,
+          doc_count: 100,
+          size_in_bytes: 10,
+        },
+        {
+          pattern_name: 'metricbeat',
+          shipper: 'metricbeat',
+          index_count: 1,
+          ecs_index_count: 0,
+          doc_count: 100,
+          size_in_bytes: 10,
+        },
+        {
+          pattern_name: 'app-search',
+          index_count: 1,
+          doc_count: 0,
+        },
+        {
+          pattern_name: 'third-party-logs',
+          index_count: 4,
+          ecs_index_count: 1,
+          doc_count: 1100,
+          size_in_bytes: 20,
+        },
+        {
+          dataset: { name: 'nginx.access', type: 'logs' },
+          shipper: 'filebeat',
+          index_count: 2,
+          ecs_index_count: 2,
+          doc_count: 2000,
+          size_in_bytes: 1060,
+        },
+      ]);
+    });
+  });
+
+  describe('getDataTelemetry', () => {
+    test('it returns the base payload (all 0s) because no indices are found', async () => {
+      const callCluster = mockCallCluster();
+      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]);
+    });
+
+    test('can only see the index mappings, but not the stats', async () => {
+      const callCluster = mockCallCluster(['filebeat-12314']);
+      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([
+        {
+          pattern_name: 'filebeat',
+          shipper: 'filebeat',
+          index_count: 1,
+          ecs_index_count: 0,
+        },
+      ]);
+    });
+
+    test('can see the mappings and the stats', async () => {
+      const callCluster = mockCallCluster(
+        ['filebeat-12314'],
+        { isECS: true },
+        {
+          indices: {
+            'filebeat-12314': { total: { docs: { count: 100 }, store: { size_in_bytes: 10 } } },
+          },
+        }
+      );
+      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([
+        {
+          pattern_name: 'filebeat',
+          shipper: 'filebeat',
+          index_count: 1,
+          ecs_index_count: 1,
+          doc_count: 100,
+          size_in_bytes: 10,
+        },
+      ]);
+    });
+
+    test('find an index that does not match any index pattern but has mappings metadata', async () => {
+      const callCluster = mockCallCluster(
+        ['cannot_match_anything'],
+        { isECS: true, datasetType: 'traces', shipper: 'my-beat' },
+        {
+          indices: {
+            cannot_match_anything: {
+              total: { docs: { count: 100 }, store: { size_in_bytes: 10 } },
+            },
+          },
+        }
+      );
+      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([
+        {
+          dataset: { name: undefined, type: 'traces' },
+          shipper: 'my-beat',
+          index_count: 1,
+          ecs_index_count: 1,
+          doc_count: 100,
+          size_in_bytes: 10,
+        },
+      ]);
+    });
+
+    test('return empty array when there is an error', async () => {
+      const callCluster = jest.fn().mockRejectedValue(new Error('Something went terribly wrong'));
+      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]);
+    });
+  });
+});
+
+function mockCallCluster(
+  indicesMappings: string[] = [],
+  { isECS = false, datasetName = '', datasetType = '', shipper = '' } = {},
+  indexStats: any = {}
+) {
+  return jest.fn().mockImplementation(async (method: string, opts: any) => {
+    if (method === 'indices.getMapping') {
+      return Object.fromEntries(
+        indicesMappings.map((index) => [
+          index,
+          {
+            mappings: {
+              ...(shipper && { _meta: { beat: shipper } }),
+              properties: {
+                ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }),
+                ...((datasetType || datasetName) && {
+                  dataset: {
+                    properties: {
+                      ...(datasetName && {
+                        name: { type: 'constant_keyword', value: datasetName },
+                      }),
+                      ...(datasetType && {
+                        type: { type: 'constant_keyword', value: datasetType },
+                      }),
+                    },
+                  },
+                }),
+              },
+            },
+          },
+        ])
+      );
+    }
+    return indexStats;
+  });
+}