[Telemetry] Validate ES and SO clients status before fetching telemetry

src/plugins/telemetry/server/fetcher.test.ts

@@ -9,21 +9,60 @@
 /* eslint-disable dot-notation */
 import { FetcherTask } from './fetcher';
 import { coreMock } from '@kbn/core/server/mocks';
+import {
+  telemetryCollectionManagerPluginMock,
+  Setup,
+} from '@kbn/telemetry-collection-manager-plugin/server/mocks';
 
 describe('FetcherTask', () => {
   describe('sendIfDue', () => {
-    it('stops when it fails to get telemetry configs', async () => {
+    let getCurrentConfigs: jest.Mock;
+    let shouldSendReport: jest.Mock;
+    let fetchTelemetry: jest.Mock;
+    let sendTelemetry: jest.Mock;
+    let updateReportFailure: jest.Mock;
+    let telemetryCollectionManagerMock: Setup;
+    let fetcherTask: FetcherTask;
+
+    beforeEach(() => {
+      getCurrentConfigs = jest.fn();
+      shouldSendReport = jest.fn();
+      fetchTelemetry = jest.fn();
+      sendTelemetry = jest.fn();
+      updateReportFailure = jest.fn();
+
       const initializerContext = coreMock.createPluginInitializerContext({});
-      const fetcherTask = new FetcherTask(initializerContext);
-      const mockError = new Error('Some message.');
-      const getCurrentConfigs = jest.fn().mockRejectedValue(mockError);
-      const fetchTelemetry = jest.fn();
-      const sendTelemetry = jest.fn();
+      fetcherTask = new FetcherTask(initializerContext);
+
+      telemetryCollectionManagerMock = telemetryCollectionManagerPluginMock.createSetupContract();
+      telemetryCollectionManagerMock.shouldGetTelemetry.mockResolvedValue(true);
+      fetcherTask['telemetryCollectionManager'] = telemetryCollectionManagerMock;
+
       Object.assign(fetcherTask, {
         getCurrentConfigs,
+        shouldSendReport,
         fetchTelemetry,
+        updateReportFailure,
         sendTelemetry,
       });
+    });
+
+    it('stops when Kibana is not ready to fetch telemetry', async () => {
+      const mockError = new Error('Some message.');
+      getCurrentConfigs.mockRejectedValue(mockError);
+      telemetryCollectionManagerMock.shouldGetTelemetry.mockResolvedValue(false);
+
+      const result = await fetcherTask['sendIfDue']();
+      expect(result).toBe(undefined);
+      expect(getCurrentConfigs).toBeCalledTimes(0);
+      expect(fetchTelemetry).toBeCalledTimes(0);
+      expect(sendTelemetry).toBeCalledTimes(0);
+      expect(fetcherTask['logger'].warn).toBeCalledTimes(0);
+    });
+
+    it('stops when it fails to get telemetry configs', async () => {
+      const mockError = new Error('Some message.');
+      getCurrentConfigs.mockRejectedValue(mockError);
       const result = await fetcherTask['sendIfDue']();
       expect(result).toBe(undefined);
       expect(getCurrentConfigs).toBeCalledTimes(1);
@@ -36,30 +75,17 @@ describe('FetcherTask', () => {
     });
 
     it('fetches usage and send telemetry', async () => {
-      const initializerContext = coreMock.createPluginInitializerContext({});
-      const fetcherTask = new FetcherTask(initializerContext);
       const mockTelemetryUrl = 'mock_telemetry_url';
       const mockClusters = [
         { clusterUuid: 'mk_uuid_1', stats: 'cluster_1' },
         { clusterUuid: 'mk_uuid_2', stats: 'cluster_2' },
       ];
 
-      const getCurrentConfigs = jest.fn().mockResolvedValue({
+      getCurrentConfigs.mockResolvedValue({
         telemetryUrl: mockTelemetryUrl,
       });
-      const shouldSendReport = jest.fn().mockReturnValue(true);
-
-      const fetchTelemetry = jest.fn().mockResolvedValue(mockClusters);
-      const sendTelemetry = jest.fn();
-      const updateReportFailure = jest.fn();
-
-      Object.assign(fetcherTask, {
-        getCurrentConfigs,
-        shouldSendReport,
-        fetchTelemetry,
-        updateReportFailure,
-        sendTelemetry,
-      });
+      shouldSendReport.mockReturnValue(true);
+      fetchTelemetry.mockResolvedValue(mockClusters);
 
       await fetcherTask['sendIfDue']();
 

src/plugins/telemetry/server/fetcher.ts

@@ -79,6 +79,12 @@ export class FetcherTask {
     if (this.isSending) {
       return;
     }
+
+    // Skip this run if Kibana is not in a healthy state to fetch telemetry.
+    if (!(await this.telemetryCollectionManager?.shouldGetTelemetry())) {
+      return;
+    }
+
     let telemetryConfig: TelemetryConfig | undefined;
 
     try {

src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts

@@ -73,6 +73,15 @@ export function registerTelemetryOptInStatsRoutes(
         const newOptInStatus = req.body.enabled;
         const unencrypted = req.body.unencrypted;
 
+        if (!(await telemetryCollectionManager.shouldGetTelemetry())) {
+          // We probably won't reach here because there is a license check in the auth phase of the HTTP requests.
+          // But let's keep it here should that changes at any point.
+          return res.customError({
+            statusCode: 503,
+            body: `Can't fetch telemetry at the moment because some services are down. Check the /status page for more details.`,
+          });
+        }
+
         const statsGetterConfig: StatsGetterConfig = {
           unencrypted,
         };

src/plugins/telemetry/server/routes/telemetry_usage_stats.test.ts

@@ -38,6 +38,7 @@ describe('registerTelemetryUsageStatsRoutes', () => {
   const mockCoreSetup = coreMock.createSetup();
   const mockStats = [{ clusterUuid: 'text', stats: 'enc_str' }];
   telemetryCollectionManager.getStats.mockResolvedValue(mockStats);
+  telemetryCollectionManager.shouldGetTelemetry.mockResolvedValue(true);
   const getSecurity = jest.fn();
 
   let mockRouter: IRouter;
@@ -119,6 +120,21 @@ describe('registerTelemetryUsageStatsRoutes', () => {
       expect(mockResponse.forbidden).toBeCalled();
     });
 
+    it('returns 503 when Kibana is not healthy enough to generate the Telemetry report', async () => {
+      telemetryCollectionManager.shouldGetTelemetry.mockResolvedValueOnce(false);
+
+      registerTelemetryUsageStatsRoutes(mockRouter, telemetryCollectionManager, true, () => void 0);
+      const { mockResponse } = await runRequest(mockRouter, {
+        refreshCache: false,
+        unencrypted: true,
+      });
+      expect(mockResponse.customError).toBeCalledTimes(1);
+      expect(mockResponse.customError).toBeCalledWith({
+        statusCode: 503,
+        body: `Can't fetch telemetry at the moment because some services are down. Check the /status page for more details.`,
+      });
+    });
+
     it('returns 200 when the user has enough permissions to request unencrypted telemetry', async () => {
       const getSecurityMock = jest.fn().mockImplementation(() => {
         const securityStartMock = securityMock.createStart();

src/plugins/telemetry/server/routes/telemetry_usage_stats.ts

@@ -35,6 +35,15 @@ export function registerTelemetryUsageStatsRoutes(
     async (context, req, res) => {
       const { unencrypted, refreshCache } = req.body;
 
+      if (!(await telemetryCollectionManager.shouldGetTelemetry())) {
+        // We probably won't reach here because there is a license check in the auth phase of the HTTP requests.
+        // But let's keep it here should that changes at any point.
+        return res.customError({
+          statusCode: 503,
+          body: `Can't fetch telemetry at the moment because some services are down. Check the /status page for more details.`,
+        });
+      }
+
       const security = getSecurity();
       if (security && unencrypted) {
         // Normally we would use `options: { tags: ['access:decryptedTelemetry'] }` in the route definition to check authorization for an

src/plugins/telemetry_collection_manager/server/mocks.ts

@@ -24,6 +24,7 @@ function createSetupContract(): Setup {
     getStats: jest.fn(),
     getOptInStats: jest.fn(),
     setCollectionStrategy: jest.fn(),
+    shouldGetTelemetry: jest.fn(),
   };
 
   return setupContract;
@@ -33,6 +34,7 @@ function createStartContract(): Start {
   const startContract: Start = {
     getOptInStats: jest.fn(),
     getStats: jest.fn(),
+    shouldGetTelemetry: jest.fn(),
   };
 
   return startContract;

src/plugins/telemetry_collection_manager/server/plugin.ts

@@ -17,8 +17,10 @@ import type {
   SavedObjectsServiceStart,
   ElasticsearchClient,
   SavedObjectsClientContract,
+  CoreStatus,
 } from '@kbn/core/server';
 
+import { firstValueFrom, ReplaySubject } from 'rxjs';
 import type {
   TelemetryCollectionManagerPluginSetup,
   TelemetryCollectionManagerPluginStart,
@@ -49,6 +51,7 @@ export class TelemetryCollectionManagerPlugin
   private readonly logger: Logger;
   private collectionStrategy: CollectionStrategy | undefined;
   private usageGetterMethodPriority = -1;
+  private coreStatus$ = new ReplaySubject<CoreStatus>(1);
   private usageCollection?: UsageCollectionSetup;
   private elasticsearchClient?: IClusterClient;
   private savedObjectsService?: SavedObjectsServiceStart;
@@ -65,10 +68,13 @@ export class TelemetryCollectionManagerPlugin
   public setup(core: CoreSetup, { usageCollection }: TelemetryCollectionPluginsDepsSetup) {
     this.usageCollection = usageCollection;
 
+    core.status.core$.subscribe(this.coreStatus$);
+
     return {
       setCollectionStrategy: this.setCollectionStrategy.bind(this),
       getOptInStats: this.getOptInStats.bind(this),
       getStats: this.getStats.bind(this),
+      shouldGetTelemetry: this.shouldGetTelemetry.bind(this),
     };
   }
 
@@ -79,10 +85,27 @@ export class TelemetryCollectionManagerPlugin
     return {
       getOptInStats: this.getOptInStats.bind(this),
       getStats: this.getStats.bind(this),
+      shouldGetTelemetry: this.shouldGetTelemetry.bind(this),
     };
   }
 
-  public stop() {}
+  public stop() {
+    this.coreStatus$.complete();
+  }
+
+  /**
+   * Checks if Kibana is in a healthy state to attempt the Telemetry report generation:
+   * - Elasticsearch is active.
+   * - SavedObjects client is active.
+   * @private
+   */
+  private async shouldGetTelemetry() {
+    const { elasticsearch, savedObjects } = await firstValueFrom(this.coreStatus$);
+    return (
+      elasticsearch.level.toString() === 'available' &&
+      savedObjects.level.toString() === 'available'
+    );
+  }
 
   private setCollectionStrategy<T extends BasicStatsPayload>(
     collectionConfig: CollectionStrategyConfig<T>

src/plugins/telemetry_collection_manager/server/types.ts

@@ -10,17 +10,35 @@ import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@k
 import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
 import type { TelemetryCollectionManagerPlugin } from './plugin';
 
-export interface TelemetryCollectionManagerPluginSetup {
+/**
+ * Public contract coming from the setup method.
+ */
+export interface TelemetryCollectionManagerPluginSetup
+  extends TelemetryCollectionManagerPluginStart {
   setCollectionStrategy: <T extends BasicStatsPayload>(
     collectionConfig: CollectionStrategyConfig<T>
   ) => void;
-  getOptInStats: TelemetryCollectionManagerPlugin['getOptInStats'];
-  getStats: TelemetryCollectionManagerPlugin['getStats'];
 }
 
+/**
+ * Public contract coming from the start method.
+ */
 export interface TelemetryCollectionManagerPluginStart {
+  /**
+   * Fetches the minimum piece of data to report when Opting IN or OUT.
+   */
   getOptInStats: TelemetryCollectionManagerPlugin['getOptInStats'];
+  /**
+   * Fetches the Snapshot telemetry report.
+   */
   getStats: TelemetryCollectionManagerPlugin['getStats'];
+  /**
+   * Is it OK to fetch telemetry?
+   *
+   * It should be called before calling `getStats` or `getOptInStats` to validate that Kibana is in a healthy state
+   * to attempt to fetch the Telemetry report.
+   */
+  shouldGetTelemetry: () => Promise<boolean>;
 }
 
 export interface TelemetryOptInStats {