From d71d94fec035c1de74999b502d7193dbb4264bf2 Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Mon, 12 Jul 2021 14:42:27 +0200 Subject: [PATCH] Skip transformations on unknown types --- .../migrations/core/migrate_raw_docs.test.ts | 87 +++++-- .../migrations/core/migrate_raw_docs.ts | 26 +- .../migrations/kibana/kibana_migrator.ts | 11 +- .../migration_7_13_0_unknown_types.test.ts | 229 +++++++++++++----- 4 files changed, 264 insertions(+), 89 deletions(-) diff --git a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts index 0481e6118acb0c..c8ff79351aadb7 100644 --- a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts +++ b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts @@ -137,14 +137,15 @@ describe('migrateRawDocsSafely', () => { const transform = jest.fn((doc: any) => [ set(_.cloneDeep(doc), 'attributes.name', 'HOI!'), ]); - const task = migrateRawDocsSafely( - new SavedObjectsSerializer(new SavedObjectTypeRegistry()), - transform, - [ + const task = migrateRawDocsSafely({ + serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()), + knownTypes: new Set(['a', 'c']), + migrateDoc: transform, + rawDocs: [ { _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }, { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } }, - ] - ); + ], + }); const result = (await task()) as Either.Right; expect(result._tag).toEqual('Right'); expect(result.right.processedDocs).toEqual([ @@ -181,14 +182,15 @@ describe('migrateRawDocsSafely', () => { const transform = jest.fn((doc: any) => [ set(_.cloneDeep(doc), 'attributes.name', 'TADA'), ]); - const task = migrateRawDocsSafely( - new SavedObjectsSerializer(new SavedObjectTypeRegistry()), - transform, - [ + const task = migrateRawDocsSafely({ + serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()), + knownTypes: new Set(['a', 'c']), + migrateDoc: transform, + rawDocs: [ { _id: 'foo:b', _source: { type: 'a', a: { name: 'AAA' } } }, { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } }, - ] - ); + ], + }); const result = (await task()) as Either.Left; expect(transform).toHaveBeenCalledTimes(1); expect(result._tag).toEqual('Left'); @@ -202,11 +204,12 @@ describe('migrateRawDocsSafely', () => { set(_.cloneDeep(doc), 'attributes.name', 'HOI!'), { id: 'bar', type: 'foo', attributes: { name: 'baz' } }, ]); - const task = migrateRawDocsSafely( - new SavedObjectsSerializer(new SavedObjectTypeRegistry()), - transform, - [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }] - ); + const task = migrateRawDocsSafely({ + serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()), + knownTypes: new Set(['a', 'c']), + migrateDoc: transform, + rawDocs: [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }], + }); const result = (await task()) as Either.Right; expect(result._tag).toEqual('Right'); expect(result.right.processedDocs).toEqual([ @@ -235,11 +238,12 @@ describe('migrateRawDocsSafely', () => { const transform = jest.fn((doc: any) => { throw new TransformSavedObjectDocumentError(new Error('error during transform'), '8.0.0'); }); - const task = migrateRawDocsSafely( - new SavedObjectsSerializer(new SavedObjectTypeRegistry()), - transform, - [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }] // this is the raw doc - ); + const task = migrateRawDocsSafely({ + serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()), + knownTypes: new Set(['a', 'c']), + migrateDoc: transform, + rawDocs: [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }], // this is the raw doc + }); const result = (await task()) as Either.Left; expect(transform).toHaveBeenCalledTimes(1); expect(result._tag).toEqual('Left'); @@ -252,4 +256,43 @@ describe('migrateRawDocsSafely', () => { } `); }); + + test('skips documents of unknown types', async () => { + const transform = jest.fn((doc: any) => [ + set(_.cloneDeep(doc), 'attributes.name', 'HOI!'), + ]); + const task = migrateRawDocsSafely({ + serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()), + knownTypes: new Set(['a']), + migrateDoc: transform, + rawDocs: [ + { _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }, + { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } }, + ], + }); + + const result = (await task()) as Either.Right; + expect(result._tag).toEqual('Right'); + expect(result.right.processedDocs).toEqual([ + { + _id: 'a:b', + _source: { type: 'a', a: { name: 'HOI!' }, migrationVersion: {}, references: [] }, + }, + { + _id: 'c:d', + // name field is not migrated on unknown type + _source: { type: 'c', c: { name: 'DDD' } }, + }, + ]); + + const obj1 = { + id: 'b', + type: 'a', + attributes: { name: 'AAA' }, + migrationVersion: {}, + references: [], + }; + expect(transform).toHaveBeenCalledTimes(1); + expect(transform).toHaveBeenNthCalledWith(1, obj1); + }); }); diff --git a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts index 461ae1df6bc3d5..65ea21a6778d5b 100644 --- a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts +++ b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts @@ -81,6 +81,13 @@ export async function migrateRawDocs( return processedDocs; } +interface MigrateRawDocsSafelyDeps { + serializer: SavedObjectsSerializer; + knownTypes: ReadonlySet; + migrateDoc: MigrateAndConvertFn; + rawDocs: SavedObjectsRawDoc[]; +} + /** * Applies the specified migration function to every saved object document provided * and converts the saved object to a raw document. @@ -88,11 +95,15 @@ export async function migrateRawDocs( * for which the transformation function failed. * @returns {TaskEither.TaskEither} */ -export function migrateRawDocsSafely( - serializer: SavedObjectsSerializer, - migrateDoc: MigrateAndConvertFn, - rawDocs: SavedObjectsRawDoc[] -): TaskEither.TaskEither { +export function migrateRawDocsSafely({ + serializer, + knownTypes, + migrateDoc, + rawDocs, +}: MigrateRawDocsSafelyDeps): TaskEither.TaskEither< + DocumentsTransformFailed, + DocumentsTransformSuccess +> { return async () => { const migrateDocNonBlocking = transformNonBlocking(migrateDoc); const processedDocs: SavedObjectsRawDoc[] = []; @@ -100,7 +111,10 @@ export function migrateRawDocsSafely( const corruptSavedObjectIds: string[] = []; const options = { namespaceTreatment: 'lax' as const }; for (const raw of rawDocs) { - if (serializer.isRawSavedObject(raw, options)) { + // Do not transform documents of unknown types + if (raw?._source?.type && !knownTypes.has(raw._source.type)) { + processedDocs.push(raw); + } else if (serializer.isRawSavedObject(raw, options)) { try { const savedObject = convertToRawAddMigrationVersion(raw, options, serializer); processedDocs.push( diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index 2d0282e6d26324..572b2934e49b82 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -184,11 +184,12 @@ export class KibanaMigrator { logger: this.log, preMigrationScript: indexMap[index].script, transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => - migrateRawDocsSafely( - this.serializer, - this.documentMigrator.migrateAndConvert, - rawDocs - ), + migrateRawDocsSafely({ + serializer: this.serializer, + knownTypes: new Set(this.typeRegistry.getAllTypes().map((t) => t.name)), + migrateDoc: this.documentMigrator.migrateAndConvert, + rawDocs, + }), migrationVersionPerType: this.documentMigrator.migrationVersion, indexPrefix: index, migrationsConfig: this.soMigrationsConfig, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts index c5e302adbe9032..a30b3d291e7ec9 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_unknown_types.test.ts @@ -7,42 +7,31 @@ */ import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; +import fs from 'fs/promises'; import { estypes } from '@elastic/elasticsearch'; import * as kbnTestServer from '../../../../test_helpers/kbn_server'; import { Root } from '../../../root'; +import JSON5 from 'json5'; +import { ElasticsearchClient } from '../../../elasticsearch'; const logFilePath = Path.join(__dirname, '7_13_unknown_types_test.log'); -const asyncUnlink = Util.promisify(Fs.unlink); - async function removeLogFile() { // ignore errors if it doesn't exist - await asyncUnlink(logFilePath).catch(() => void 0); + await fs.unlink(logFilePath).catch(() => void 0); } describe('migration v2', () => { let esServer: kbnTestServer.TestElasticsearchUtils; let root: Root; + let startES: () => Promise; beforeAll(async () => { await removeLogFile(); }); - afterAll(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - - await new Promise((resolve) => setTimeout(resolve, 10000)); - }); - - it('migrates the documents to the highest version', async () => { - const { startES } = kbnTestServer.createTestServers({ + beforeEach(() => { + ({ startES } = kbnTestServer.createTestServers({ adjustTimeout: (t: number) => jest.setTimeout(t), settings: { es: { @@ -53,50 +42,155 @@ describe('migration v2', () => { dataArchive: Path.join(__dirname, 'archives', '7.13.0_with_unknown_so.zip'), }, }, - }); + })); + }); + + afterEach(async () => { + if (root) { + await root.shutdown(); + } + if (esServer) { + await esServer.stop(); + } + + await new Promise((resolve) => setTimeout(resolve, 10000)); + }); + it('logs a warning and completes the migration with unknown docs retained', async () => { root = createRoot(); + esServer = await startES(); + await root.setup(); + await root.start(); + + const logFileContent = await fs.readFile(logFilePath, 'utf-8'); + const records = logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)); + + const unknownDocsWarningLog = records.find((rec) => + rec.message.startsWith(`[.kibana] CHECK_UNKNOWN_DOCUMENTS`) + ); + + expect( + unknownDocsWarningLog.message.startsWith( + '[.kibana] CHECK_UNKNOWN_DOCUMENTS Upgrades will fail for 8.0+ because documents were found for unknown saved ' + + 'object types. To ensure that upgrades will succeed in the future, either re-enable plugins or delete ' + + 'these documents from the ".kibana_8.0.0_001" index after the current upgrade completes.' + ) + ).toBeTruthy(); + + const unknownDocs = [ + { type: 'space', id: 'space:default' }, + { type: 'space', id: 'space:first' }, + { type: 'space', id: 'space:second' }, + { type: 'space', id: 'space:third' }, + { type: 'space', id: 'space:forth' }, + { type: 'space', id: 'space:fifth' }, + { type: 'space', id: 'space:sixth' }, + { type: 'foo', id: 'P2SQfHkBs3dBRGh--No5' }, + { type: 'foo', id: 'QGSZfHkBs3dBRGh-ANoD' }, + { type: 'foo', id: 'QWSZfHkBs3dBRGh-hNob' }, + ]; + + unknownDocs.forEach(({ id, type }) => { + expect(unknownDocsWarningLog.message).toEqual( + expect.stringContaining(`- "${id}" (type: "${type}")`) + ); + }); + + const client: ElasticsearchClient = esServer.es.getClient(); + const { body: response } = await client.indices.getSettings({ index: '.kibana_8.0.0_001' }); + const settings = response['.kibana_8.0.0_001'] + .settings as estypes.IndicesIndexStatePrefixedSettings; + expect(settings.index).not.toBeUndefined(); + expect(settings.index!.blocks?.write).not.toEqual('true'); + + // Ensure that documents for unknown types were preserved in target index in an unmigrated state + const spaceDocs = await fetchDocs(client, '.kibana_8.0.0_001', 'space'); + expect(spaceDocs.map((s) => s.id)).toEqual( + expect.arrayContaining([ + 'space:default', + 'space:first', + 'space:second', + 'space:third', + 'space:forth', + 'space:fifth', + 'space:sixth', + ]) + ); + spaceDocs.forEach((d) => { + expect(d.migrationVersion.space).toEqual('6.6.0'); + expect(d.coreMigrationVersion).toEqual('7.13.0'); + }); + const fooDocs = await fetchDocs(client, '.kibana_8.0.0_001', 'foo'); + expect(fooDocs.map((f) => f.id)).toEqual( + expect.arrayContaining([ + 'P2SQfHkBs3dBRGh--No5', + 'QGSZfHkBs3dBRGh-ANoD', + 'QWSZfHkBs3dBRGh-hNob', + ]) + ); + fooDocs.forEach((d) => { + expect(d.migrationVersion.foo).toEqual('7.13.0'); + expect(d.coreMigrationVersion).toEqual('7.13.0'); + }); + }); + it('migrates outdated documents when types are re-enabled', async () => { + // Start kibana with foo and space types disabled + root = createRoot(); esServer = await startES(); await root.setup(); + await root.start(); - try { - await root.start(); - } catch (err) { - const errorMessage = err.message; - - expect( - errorMessage.startsWith( - 'Unable to complete saved object migrations for the [.kibana] index: Migration failed because documents ' + - 'were found for unknown saved object types. To proceed with the migration, please delete these documents from the ' + - '".kibana_7.13.0_001" index.' - ) - ).toBeTruthy(); - - const unknownDocs = [ - { type: 'space', id: 'space:default' }, - { type: 'space', id: 'space:first' }, - { type: 'space', id: 'space:second' }, - { type: 'space', id: 'space:third' }, - { type: 'space', id: 'space:forth' }, - { type: 'space', id: 'space:fifth' }, - { type: 'space', id: 'space:sixth' }, - { type: 'foo', id: 'P2SQfHkBs3dBRGh--No5' }, - { type: 'foo', id: 'QGSZfHkBs3dBRGh-ANoD' }, - { type: 'foo', id: 'QWSZfHkBs3dBRGh-hNob' }, - ]; - - unknownDocs.forEach(({ id, type }) => { - expect(errorMessage).toEqual(expect.stringContaining(`- "${id}" (type: "${type}")`)); - }); - - const client = esServer.es.getClient(); - const { body: response } = await client.indices.getSettings({ index: '.kibana_7.13.0_001' }); - const settings = response['.kibana_7.13.0_001'] - .settings as estypes.IndicesIndexStatePrefixedSettings; - expect(settings.index).not.toBeUndefined(); - expect(settings.index!.blocks?.write).not.toEqual('true'); - } + // Shutdown and start Kibana again with space type registered to ensure space docs get migrated + await root.shutdown(); + root = createRoot(); + const coreSetup = await root.setup(); + coreSetup.savedObjects.registerType({ + name: 'space', + hidden: false, + mappings: { properties: {} }, + namespaceType: 'agnostic', + migrations: { + '6.6.0': (d) => d, + '8.0.0': (d) => d, + }, + }); + await root.start(); + + const client: ElasticsearchClient = esServer.es.getClient(); + const spacesDocsMigrated = await fetchDocs(client, '.kibana_8.0.0_001', 'space'); + expect(spacesDocsMigrated.map((s) => s.id)).toEqual( + expect.arrayContaining([ + 'space:default', + 'space:first', + 'space:second', + 'space:third', + 'space:forth', + 'space:fifth', + 'space:sixth', + ]) + ); + spacesDocsMigrated.forEach((d) => { + expect(d.migrationVersion.space).toEqual('8.0.0'); // should be migrated + expect(d.coreMigrationVersion).toEqual('8.0.0'); + }); + + // Make sure unmigrated foo docs are also still there in an unmigrated state + const fooDocsUnmigrated = await fetchDocs(client, '.kibana_8.0.0_001', 'foo'); + expect(fooDocsUnmigrated.map((f) => f.id)).toEqual( + expect.arrayContaining([ + 'P2SQfHkBs3dBRGh--No5', + 'QGSZfHkBs3dBRGh-ANoD', + 'QWSZfHkBs3dBRGh-hNob', + ]) + ); + fooDocsUnmigrated.forEach((d) => { + expect(d.migrationVersion.foo).toEqual('7.13.0'); // should still not be migrated + expect(d.coreMigrationVersion).toEqual('7.13.0'); + }); }); }); @@ -131,3 +225,26 @@ function createRoot() { } ); } + +async function fetchDocs(esClient: ElasticsearchClient, index: string, type: string) { + const { body } = await esClient.search({ + index, + size: 10000, + body: { + query: { + bool: { + should: [ + { + term: { type }, + }, + ], + }, + }, + }, + }); + + return body.hits.hits.map((h) => ({ + ...h._source, + id: h._id, + })); +}