Add deprecation warning when unknown SO types are present (elastic#11…

…1268)

* Add deprecation warning when unknown types are present

* fix and add service tests

* remove export

* plug deprecation route

* add integration test for new route

* add unit test for getIndexForType

* add unit tests

* improve deprecation messages

* add FTR test

* fix things due to merge

* change the name of the deprecation provider

* improve message

* improve message again
# Conflicts:
#	src/core/server/saved_objects/saved_objects_service.ts

src/core/server/saved_objects/deprecations/deprecation_factory.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { RegisterDeprecationsConfig } from '../../deprecations';
+import type { ISavedObjectTypeRegistry } from '../saved_objects_type_registry';
+import type { SavedObjectConfig } from '../saved_objects_config';
+import type { KibanaConfigType } from '../../kibana_config';
+import { getUnknownTypesDeprecations } from './unknown_object_types';
+
+interface GetDeprecationProviderOptions {
+  typeRegistry: ISavedObjectTypeRegistry;
+  savedObjectsConfig: SavedObjectConfig;
+  kibanaConfig: KibanaConfigType;
+  kibanaVersion: string;
+}
+
+export const getSavedObjectsDeprecationsProvider = (
+  config: GetDeprecationProviderOptions
+): RegisterDeprecationsConfig => {
+  return {
+    getDeprecations: async (context) => {
+      return [
+        ...(await getUnknownTypesDeprecations({
+          ...config,
+          esClient: context.esClient,
+        })),
+      ];
+    },
+  };
+};

src/core/server/saved_objects/deprecations/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { getSavedObjectsDeprecationsProvider } from './deprecation_factory';
+export { deleteUnknownTypeObjects } from './unknown_object_types';

src/core/server/saved_objects/deprecations/unknown_object_types.test.mocks.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const getIndexForTypeMock = jest.fn();
+
+jest.doMock('../service/lib/get_index_for_type', () => ({
+  getIndexForType: getIndexForTypeMock,
+}));

src/core/server/saved_objects/deprecations/unknown_object_types.test.ts

@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getIndexForTypeMock } from './unknown_object_types.test.mocks';
+
+import { estypes } from '@elastic/elasticsearch';
+import { deleteUnknownTypeObjects, getUnknownTypesDeprecations } from './unknown_object_types';
+import { typeRegistryMock } from '../saved_objects_type_registry.mock';
+import { elasticsearchClientMock } from '../../elasticsearch/client/mocks';
+import type { KibanaConfigType } from '../../kibana_config';
+import type { SavedObjectConfig } from '../saved_objects_config';
+import { SavedObjectsType } from 'kibana/server';
+
+const createSearchResponse = (count: number): estypes.SearchResponse => {
+  return {
+    hits: {
+      total: count,
+      max_score: 0,
+      hits: new Array(count).fill({}),
+    },
+  } as estypes.SearchResponse;
+};
+
+describe('unknown saved object types deprecation', () => {
+  const kibanaVersion = '8.0.0';
+
+  let typeRegistry: ReturnType<typeof typeRegistryMock.create>;
+  let esClient: ReturnType<typeof elasticsearchClientMock.createScopedClusterClient>;
+  let kibanaConfig: KibanaConfigType;
+  let savedObjectsConfig: SavedObjectConfig;
+
+  beforeEach(() => {
+    typeRegistry = typeRegistryMock.create();
+    esClient = elasticsearchClientMock.createScopedClusterClient();
+
+    typeRegistry.getAllTypes.mockReturnValue([
+      { name: 'foo' },
+      { name: 'bar' },
+    ] as SavedObjectsType[]);
+    getIndexForTypeMock.mockImplementation(({ type }: { type: string }) => `${type}-index`);
+
+    kibanaConfig = {
+      index: '.kibana',
+      enabled: true,
+    };
+
+    savedObjectsConfig = {
+      migration: {
+        enableV2: true,
+      },
+    } as SavedObjectConfig;
+  });
+
+  afterEach(() => {
+    getIndexForTypeMock.mockReset();
+  });
+
+  describe('getUnknownTypesDeprecations', () => {
+    beforeEach(() => {
+      esClient.asInternalUser.search.mockReturnValue(
+        elasticsearchClientMock.createSuccessTransportRequestPromise(createSearchResponse(0))
+      );
+    });
+
+    it('calls `esClient.asInternalUser.search` with the correct parameters', async () => {
+      await getUnknownTypesDeprecations({
+        savedObjectsConfig,
+        esClient,
+        typeRegistry,
+        kibanaConfig,
+        kibanaVersion,
+      });
+
+      expect(esClient.asInternalUser.search).toHaveBeenCalledTimes(1);
+      expect(esClient.asInternalUser.search).toHaveBeenCalledWith({
+        index: ['foo-index', 'bar-index'],
+        body: {
+          size: 10000,
+          query: {
+            bool: {
+              must_not: [{ term: { type: 'foo' } }, { term: { type: 'bar' } }],
+            },
+          },
+        },
+      });
+    });
+
+    it('returns no deprecation if no unknown type docs are found', async () => {
+      esClient.asInternalUser.search.mockReturnValue(
+        elasticsearchClientMock.createSuccessTransportRequestPromise(createSearchResponse(0))
+      );
+
+      const deprecations = await getUnknownTypesDeprecations({
+        savedObjectsConfig,
+        esClient,
+        typeRegistry,
+        kibanaConfig,
+        kibanaVersion,
+      });
+
+      expect(deprecations.length).toEqual(0);
+    });
+
+    it('returns a deprecation if any unknown type docs are found', async () => {
+      esClient.asInternalUser.search.mockReturnValue(
+        elasticsearchClientMock.createSuccessTransportRequestPromise(createSearchResponse(1))
+      );
+
+      const deprecations = await getUnknownTypesDeprecations({
+        savedObjectsConfig,
+        esClient,
+        typeRegistry,
+        kibanaConfig,
+        kibanaVersion,
+      });
+
+      expect(deprecations.length).toEqual(1);
+      expect(deprecations[0]).toEqual({
+        title: expect.any(String),
+        message: expect.any(String),
+        level: 'critical',
+        requireRestart: false,
+        deprecationType: undefined,
+        correctiveActions: {
+          manualSteps: expect.any(Array),
+          api: {
+            path: '/internal/saved_objects/deprecations/_delete_unknown_types',
+            method: 'POST',
+            body: {},
+          },
+        },
+      });
+    });
+  });
+
+  describe('deleteUnknownTypeObjects', () => {
+    it('calls `esClient.asInternalUser.search` with the correct parameters', async () => {
+      await deleteUnknownTypeObjects({
+        savedObjectsConfig,
+        esClient,
+        typeRegistry,
+        kibanaConfig,
+        kibanaVersion,
+      });
+
+      expect(esClient.asInternalUser.deleteByQuery).toHaveBeenCalledTimes(1);
+      expect(esClient.asInternalUser.deleteByQuery).toHaveBeenCalledWith({
+        index: ['foo-index', 'bar-index'],
+        wait_for_completion: false,
+        body: {
+          query: {
+            bool: {
+              must_not: [{ term: { type: 'foo' } }, { term: { type: 'bar' } }],
+            },
+          },
+        },
+      });
+    });
+  });
+});

src/core/server/saved_objects/deprecations/unknown_object_types.ts

@@ -0,0 +1,172 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+import { i18n } from '@kbn/i18n';
+import type { DeprecationsDetails } from '../../deprecations';
+import { IScopedClusterClient } from '../../elasticsearch';
+import { ISavedObjectTypeRegistry } from '../saved_objects_type_registry';
+import { SavedObjectsRawDocSource } from '../serialization';
+import type { KibanaConfigType } from '../../kibana_config';
+import type { SavedObjectConfig } from '../saved_objects_config';
+import { getIndexForType } from '../service/lib';
+
+interface UnknownTypesDeprecationOptions {
+  typeRegistry: ISavedObjectTypeRegistry;
+  esClient: IScopedClusterClient;
+  kibanaConfig: KibanaConfigType;
+  savedObjectsConfig: SavedObjectConfig;
+  kibanaVersion: string;
+}
+
+const getKnownTypes = (typeRegistry: ISavedObjectTypeRegistry) =>
+  typeRegistry.getAllTypes().map((type) => type.name);
+
+const getTargetIndices = ({
+  types,
+  typeRegistry,
+  kibanaVersion,
+  kibanaConfig,
+  savedObjectsConfig,
+}: {
+  types: string[];
+  typeRegistry: ISavedObjectTypeRegistry;
+  savedObjectsConfig: SavedObjectConfig;
+  kibanaConfig: KibanaConfigType;
+  kibanaVersion: string;
+}) => {
+  return [
+    ...new Set(
+      types.map((type) =>
+        getIndexForType({
+          type,
+          typeRegistry,
+          migV2Enabled: savedObjectsConfig.migration.enableV2,
+          kibanaVersion,
+          defaultIndex: kibanaConfig.index,
+        })
+      )
+    ),
+  ];
+};
+
+const getUnknownTypesQuery = (knownTypes: string[]): estypes.QueryDslQueryContainer => {
+  return {
+    bool: {
+      must_not: knownTypes.map((type) => ({
+        term: { type },
+      })),
+    },
+  };
+};
+
+const getUnknownSavedObjects = async ({
+  typeRegistry,
+  esClient,
+  kibanaConfig,
+  savedObjectsConfig,
+  kibanaVersion,
+}: UnknownTypesDeprecationOptions) => {
+  const knownTypes = getKnownTypes(typeRegistry);
+  const targetIndices = getTargetIndices({
+    types: knownTypes,
+    typeRegistry,
+    kibanaConfig,
+    kibanaVersion,
+    savedObjectsConfig,
+  });
+  const query = getUnknownTypesQuery(knownTypes);
+
+  const { body } = await esClient.asInternalUser.search<SavedObjectsRawDocSource>({
+    index: targetIndices,
+    body: {
+      size: 10000,
+      query,
+    },
+  });
+  const { hits: unknownDocs } = body.hits;
+
+  return unknownDocs.map((doc) => ({ id: doc._id, type: doc._source?.type ?? 'unknown' }));
+};
+
+export const getUnknownTypesDeprecations = async (
+  options: UnknownTypesDeprecationOptions
+): Promise<DeprecationsDetails[]> => {
+  const deprecations: DeprecationsDetails[] = [];
+  const unknownDocs = await getUnknownSavedObjects(options);
+  if (unknownDocs.length) {
+    deprecations.push({
+      title: i18n.translate('core.savedObjects.deprecations.unknownTypes.title', {
+        defaultMessage: 'Saved objects with unknown types are present in Kibana system indices',
+      }),
+      message: i18n.translate('core.savedObjects.deprecations.unknownTypes.message', {
+        defaultMessage:
+          '{objectCount, plural, one {# object} other {# objects}} with unknown types {objectCount, plural, one {was} other {were}} found in Kibana system indices. ' +
+          'Upgrading with unknown savedObject types is no longer supported. ' +
+          `To ensure that upgrades will succeed in the future, either re-enable plugins or delete these documents from the Kibana indices`,
+        values: {
+          objectCount: unknownDocs.length,
+        },
+      }),
+      level: 'critical',
+      requireRestart: false,
+      deprecationType: undefined, // not config nor feature...
+      correctiveActions: {
+        manualSteps: [
+          i18n.translate('core.savedObjects.deprecations.unknownTypes.manualSteps.1', {
+            defaultMessage: 'Enable disabled plugins then restart Kibana.',
+          }),
+          i18n.translate('core.savedObjects.deprecations.unknownTypes.manualSteps.2', {
+            defaultMessage:
+              'If no plugins are disabled, or if enabling them does not fix the issue, delete the documents.',
+          }),
+        ],
+        api: {
+          path: '/internal/saved_objects/deprecations/_delete_unknown_types',
+          method: 'POST',
+          body: {},
+        },
+      },
+    });
+  }
+  return deprecations;
+};
+
+interface DeleteUnknownTypesOptions {
+  typeRegistry: ISavedObjectTypeRegistry;
+  esClient: IScopedClusterClient;
+  kibanaConfig: KibanaConfigType;
+  savedObjectsConfig: SavedObjectConfig;
+  kibanaVersion: string;
+}
+
+export const deleteUnknownTypeObjects = async ({
+  esClient,
+  typeRegistry,
+  kibanaConfig,
+  savedObjectsConfig,
+  kibanaVersion,
+}: DeleteUnknownTypesOptions) => {
+  const knownTypes = getKnownTypes(typeRegistry);
+  const targetIndices = getTargetIndices({
+    types: knownTypes,
+    typeRegistry,
+    kibanaConfig,
+    kibanaVersion,
+    savedObjectsConfig,
+  });
+  const query = getUnknownTypesQuery(knownTypes);
+
+  await esClient.asInternalUser.deleteByQuery({
+    index: targetIndices,
+    wait_for_completion: false,
+    body: {
+      query,
+    },
+  });
+};