Skip to content

Commit

Permalink
[7.x] Log a warning when documents of unknown types are detected duri…
Browse files Browse the repository at this point in the history 
…ng migration (#105213) (#105414)

* Log a warning when documents of unknown types are detected during migration (#105213)

* Parameterize test for kibana version
  • Loading branch information
@joshdover
joshdover committed Jul 13, 2021
1 parent 6593a06 commit ce6a29d
Show file tree
Hide file tree
Showing 13 changed files with 353 additions and 134 deletions.
87 changes: 65 additions & 22 deletions src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,14 +137,15 @@ describe('migrateRawDocsSafely', () => {
const transform = jest.fn<any, any>((doc: any) => [
set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
]);
const task = migrateRawDocsSafely(
new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
transform,
[
const task = migrateRawDocsSafely({
serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
knownTypes: new Set(['a', 'c']),
migrateDoc: transform,
rawDocs: [
{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } },
{ _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
]
);
],
});
const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
expect(result._tag).toEqual('Right');
expect(result.right.processedDocs).toEqual([
Expand Down Expand Up @@ -181,14 +182,15 @@ describe('migrateRawDocsSafely', () => {
const transform = jest.fn<any, any>((doc: any) => [
set(_.cloneDeep(doc), 'attributes.name', 'TADA'),
]);
const task = migrateRawDocsSafely(
new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
transform,
[
const task = migrateRawDocsSafely({
serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
knownTypes: new Set(['a', 'c']),
migrateDoc: transform,
rawDocs: [
{ _id: 'foo:b', _source: { type: 'a', a: { name: 'AAA' } } },
{ _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
]
);
],
});
const result = (await task()) as Either.Left<DocumentsTransformFailed>;
expect(transform).toHaveBeenCalledTimes(1);
expect(result._tag).toEqual('Left');
Expand All @@ -202,11 +204,12 @@ describe('migrateRawDocsSafely', () => {
set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
{ id: 'bar', type: 'foo', attributes: { name: 'baz' } },
]);
const task = migrateRawDocsSafely(
new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
transform,
[{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }]
);
const task = migrateRawDocsSafely({
serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
knownTypes: new Set(['a', 'c']),
migrateDoc: transform,
rawDocs: [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }],
});
const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
expect(result._tag).toEqual('Right');
expect(result.right.processedDocs).toEqual([
Expand Down Expand Up @@ -235,11 +238,12 @@ describe('migrateRawDocsSafely', () => {
const transform = jest.fn<any, any>((doc: any) => {
throw new TransformSavedObjectDocumentError(new Error('error during transform'), '8.0.0');
});
const task = migrateRawDocsSafely(
new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
transform,
[{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }] // this is the raw doc
);
const task = migrateRawDocsSafely({
serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
knownTypes: new Set(['a', 'c']),
migrateDoc: transform,
rawDocs: [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }], // this is the raw doc
});
const result = (await task()) as Either.Left<DocumentsTransformFailed>;
expect(transform).toHaveBeenCalledTimes(1);
expect(result._tag).toEqual('Left');
Expand All @@ -252,4 +256,43 @@ describe('migrateRawDocsSafely', () => {
}
`);
});

test('skips documents of unknown types', async () => {
const transform = jest.fn<any, any>((doc: any) => [
set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
]);
const task = migrateRawDocsSafely({
serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
knownTypes: new Set(['a']),
migrateDoc: transform,
rawDocs: [
{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } },
{ _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
],
});

const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
expect(result._tag).toEqual('Right');
expect(result.right.processedDocs).toEqual([
{
_id: 'a:b',
_source: { type: 'a', a: { name: 'HOI!' }, migrationVersion: {}, references: [] },
},
{
_id: 'c:d',
// name field is not migrated on unknown type
_source: { type: 'c', c: { name: 'DDD' } },
},
]);

const obj1 = {
id: 'b',
type: 'a',
attributes: { name: 'AAA' },
migrationVersion: {},
references: [],
};
expect(transform).toHaveBeenCalledTimes(1);
expect(transform).toHaveBeenNthCalledWith(1, obj1);
});
});
26 changes: 20 additions & 6 deletions src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,26 +81,40 @@ export async function migrateRawDocs(
return processedDocs;
}

interface MigrateRawDocsSafelyDeps {
serializer: SavedObjectsSerializer;
knownTypes: ReadonlySet<string>;
migrateDoc: MigrateAndConvertFn;
rawDocs: SavedObjectsRawDoc[];
}

/**
* Applies the specified migration function to every saved object document provided
* and converts the saved object to a raw document.
* Captures the ids and errors from any documents that are not valid saved objects or
* for which the transformation function failed.
* @returns {TaskEither.TaskEither<DocumentsTransformFailed, DocumentsTransformSuccess>}
*/
export function migrateRawDocsSafely(
serializer: SavedObjectsSerializer,
migrateDoc: MigrateAndConvertFn,
rawDocs: SavedObjectsRawDoc[]
): TaskEither.TaskEither<DocumentsTransformFailed, DocumentsTransformSuccess> {
export function migrateRawDocsSafely({
serializer,
knownTypes,
migrateDoc,
rawDocs,
}: MigrateRawDocsSafelyDeps): TaskEither.TaskEither<
DocumentsTransformFailed,
DocumentsTransformSuccess
> {
return async () => {
const migrateDocNonBlocking = transformNonBlocking(migrateDoc);
const processedDocs: SavedObjectsRawDoc[] = [];
const transformErrors: TransformErrorObjects[] = [];
const corruptSavedObjectIds: string[] = [];
const options = { namespaceTreatment: 'lax' as const };
for (const raw of rawDocs) {
if (serializer.isRawSavedObject(raw, options)) {
// Do not transform documents of unknown types
if (raw?._source?.type && !knownTypes.has(raw._source.type)) {
processedDocs.push(raw);
} else if (serializer.isRawSavedObject(raw, options)) {
try {
const savedObject = convertToRawAddMigrationVersion(raw, options, serializer);
processedDocs.push(
Expand Down
11 changes: 6 additions & 5 deletions src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,11 +184,12 @@ export class KibanaMigrator {
logger: this.log,
preMigrationScript: indexMap[index].script,
transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) =>
migrateRawDocsSafely(
this.serializer,
this.documentMigrator.migrateAndConvert,
rawDocs
),
migrateRawDocsSafely({
serializer: this.serializer,
knownTypes: new Set(this.typeRegistry.getAllTypes().map((t) => t.name)),
migrateDoc: this.documentMigrator.migrateAndConvert,
rawDocs,
}),
migrationVersionPerType: this.documentMigrator.migrationVersion,
indexPrefix: index,
migrationsConfig: this.soMigrationsConfig,
Expand Down
15 changes: 8 additions & 7 deletions src/core/server/saved_objects/migrationsv2/actions/check_for_unknown_docs.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,12 @@ describe('checkForUnknownDocs', () => {
const result = await task();

expect(Either.isRight(result)).toBe(true);
expect((result as Either.Right<any>).right).toEqual({
unknownDocs: [],
});
});

it('resolves with `Either.left` when unknown docs are found', async () => {
it('resolves with `Either.right` when unknown docs are found', async () => {
const client = elasticsearchClientMock.createInternalClient(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: {
Expand All @@ -120,9 +123,8 @@ describe('checkForUnknownDocs', () => {

const result = await task();

expect(Either.isLeft(result)).toBe(true);
expect((result as Either.Left<any>).left).toEqual({
type: 'unknown_docs_found',
expect(Either.isRight(result)).toBe(true);
expect((result as Either.Right<any>).right).toEqual({
unknownDocs: [
{ id: '12', type: 'foo' },
{ id: '14', type: 'bar' },
Expand All @@ -148,9 +150,8 @@ describe('checkForUnknownDocs', () => {

const result = await task();

expect(Either.isLeft(result)).toBe(true);
expect((result as Either.Left<any>).left).toEqual({
type: 'unknown_docs_found',
expect(Either.isRight(result)).toBe(true);
expect((result as Either.Right<any>).right).toEqual({
unknownDocs: [{ id: '12', type: 'unknown' }],
});
});
Expand Down
16 changes: 5 additions & 11 deletions src/core/server/saved_objects/migrationsv2/actions/check_for_unknown_docs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ export interface CheckForUnknownDocsFoundDoc {

/** @internal */
export interface UnknownDocsFound {
type: 'unknown_docs_found';
unknownDocs: CheckForUnknownDocsFoundDoc[];
}

Expand All @@ -42,8 +41,8 @@ export const checkForUnknownDocs = ({
unusedTypesQuery,
knownTypes,
}: CheckForUnknownDocsParams): TaskEither.TaskEither<
RetryableEsClientError | UnknownDocsFound,
{}
RetryableEsClientError,
UnknownDocsFound
> => () => {
const query = createUnknownDocQuery(unusedTypesQuery, knownTypes);

Expand All @@ -56,14 +55,9 @@ export const checkForUnknownDocs = ({
})
.then((response) => {
const { hits } = response.body.hits;
if (hits.length) {
return Either.left({
type: 'unknown_docs_found' as const,
unknownDocs: hits.map((hit) => ({ id: hit._id, type: hit._source?.type ?? 'unknown' })),
});
} else {
return Either.right({});
}
return Either.right({
unknownDocs: hits.map((hit) => ({ id: hit._id, type: hit._source?.type ?? 'unknown' })),
});
})
.catch(catchRetryableEsClientErrors);
};
Expand Down
2 changes: 0 additions & 2 deletions src/core/server/saved_objects/migrationsv2/actions/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ export type {
} from './update_and_pickup_mappings';
export { updateAndPickupMappings } from './update_and_pickup_mappings';

import type { UnknownDocsFound } from './check_for_unknown_docs';
export type {
CheckForUnknownDocsParams,
UnknownDocsFound,
Expand Down Expand Up @@ -131,7 +130,6 @@ export interface ActionErrorTypeMap {
alias_not_found_exception: AliasNotFound;
remove_index_not_a_concrete_index: RemoveIndexNotAConcreteIndex;
documents_transform_failed: DocumentsTransformFailed;
unknown_docs_found: UnknownDocsFound;
}

/**
Expand Down
Loading

0 comments on commit ce6a29d

Please sign in to comment.