Skip to content

Commit

Permalink
migrationsv2 correctly handle unknown saved object type mappings
Browse files Browse the repository at this point in the history
  • Loading branch information
@rudolf
rudolf committed Jan 28, 2021
1 parent 329b9c6 commit 9baf383
Show file tree
Hide file tree
Showing 2 changed files with 188 additions and 34 deletions.
202 changes: 178 additions & 24 deletions src/core/server/saved_objects/migrationsv2/model.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,45 +182,49 @@ describe('migrations v2 model', () => {
versionAlias: '.kibana_7.11.0',
versionIndex: '.kibana_7.11.0_001',
};
const mappingsWithUnknownType = {
properties: {
disabled_saved_object_type: {
properties: {
value: { type: 'keyword' },
},
},
},
_meta: {
migrationMappingPropertyHashes: {
disabled_saved_object_type: '7997cf5a56cc02bdc9c93361bde732b0',
},
},
};

test('INIT -> OUTDATED_DOCUMENTS_SEARCH if .kibana is already pointing to the target index', () => {
const res: ResponseType<'INIT'> = Either.right({
'.kibana_7.11.0_001': {
aliases: {
'.kibana': {},
'.kibana_7.11.0': {},
},
mappings: {
properties: {
disabled_saved_object_type: {
properties: {
value: { type: 'keyword' },
},
},
},
_meta: {
migrationMappingPropertyHashes: {
disabled_saved_object_type: '7997cf5a56cc02bdc9c93361bde732b0',
},
},
},
mappings: mappingsWithUnknownType,
settings: {},
},
});
const newState = model(initState, res);

expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
// This snapshot asserts that we merge the
// migrationMappingPropertyHashes of the existing index, but we leave
// the mappings for the disabled_saved_object_type untouched. There
// might be another Kibana instance that knows about this type and
// needs these mappings in place.
expect(newState.targetIndexMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"disabled_saved_object_type": "7997cf5a56cc02bdc9c93361bde732b0",
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
Expand Down Expand Up @@ -271,7 +275,7 @@ describe('migrations v2 model', () => {
'.kibana': {},
'.kibana_7.12.0': {},
},
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
mappings: mappingsWithUnknownType,
settings: {},
},
'.kibana_7.11.0_001': {
Expand All @@ -288,12 +292,37 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('.kibana_7.invalid.0_001'),
targetIndex: '.kibana_7.11.0_001',
});
// This snapshot asserts that we disable the unknown saved object
// type. Because it's mappings are disabled, we also don't copy the
// `_meta.migrationMappingPropertyHashes` for the disabled type.
expect(newState.targetIndexMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
},
},
}
`);
});
test('INIT -> SET_SOURCE_WRITE_BLOCK when migrating from a v2 migrations index (>= 7.11.0)', () => {
const res: ResponseType<'INIT'> = Either.right({
'.kibana_7.11.0_001': {
aliases: { '.kibana': {}, '.kibana_7.11.0': {} },
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
mappings: mappingsWithUnknownType,
settings: {},
},
'.kibana_3': {
Expand All @@ -319,6 +348,31 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('.kibana_7.11.0_001'),
targetIndex: '.kibana_7.12.0_001',
});
// This snapshot asserts that we disable the unknown saved object
// type. Because it's mappings are disabled, we also don't copy the
// `_meta.migrationMappingPropertyHashes` for the disabled type.
expect(newState.targetIndexMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
},
},
}
`);
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand All @@ -328,7 +382,7 @@ describe('migrations v2 model', () => {
aliases: {
'.kibana': {},
},
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
mappings: mappingsWithUnknownType,
settings: {},
},
});
Expand All @@ -339,14 +393,39 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('.kibana_3'),
targetIndex: '.kibana_7.11.0_001',
});
// This snapshot asserts that we disable the unknown saved object
// type. Because it's mappings are disabled, we also don't copy the
// `_meta.migrationMappingPropertyHashes` for the disabled type.
expect(newState.targetIndexMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
},
},
}
`);
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
test('INIT -> LEGACY_SET_WRITE_BLOCK when migrating from a legacy index (>= 6.0.0 < 6.5)', () => {
const res: ResponseType<'INIT'> = Either.right({
'.kibana': {
aliases: {},
mappings: { properties: {}, _meta: {} },
mappings: mappingsWithUnknownType,
settings: {},
},
});
Expand All @@ -357,6 +436,31 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('.kibana_pre6.5.0_001'),
targetIndex: '.kibana_7.11.0_001',
});
// This snapshot asserts that we disable the unknown saved object
// type. Because it's mappings are disabled, we also don't copy the
// `_meta.migrationMappingPropertyHashes` for the disabled type.
expect(newState.targetIndexMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
},
},
}
`);
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand All @@ -366,7 +470,7 @@ describe('migrations v2 model', () => {
aliases: {
'my-saved-objects': {},
},
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
mappings: mappingsWithUnknownType,
settings: {},
},
});
Expand All @@ -386,6 +490,31 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('my-saved-objects_3'),
targetIndex: 'my-saved-objects_7.11.0_001',
});
// This snapshot asserts that we disable the unknown saved object
// type. Because it's mappings are disabled, we also don't copy the
// `_meta.migrationMappingPropertyHashes` for the disabled type.
expect(newState.targetIndexMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
},
},
}
`);
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand All @@ -395,7 +524,7 @@ describe('migrations v2 model', () => {
aliases: {
'my-saved-objects': {},
},
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
mappings: mappingsWithUnknownType,
settings: {},
},
});
Expand All @@ -416,6 +545,31 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('my-saved-objects_7.11.0'),
targetIndex: 'my-saved-objects_7.12.0_001',
});
// This snapshot asserts that we disable the unknown saved object
// type. Because it's mappings are disabled, we also don't copy the
// `_meta.migrationMappingPropertyHashes` for the disabled type.
expect(newState.targetIndexMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
},
},
}
`);
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand Down
20 changes: 10 additions & 10 deletions src/core/server/saved_objects/migrationsv2/model.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,13 @@ function throwBadResponse(state: State, res: any): never {
* Merge the _meta.migrationMappingPropertyHashes mappings of an index with
* the given target mappings.
*
* @remarks Mapping updates are commutative (deeply merged) by Elasticsearch,
* except for the _meta key. The source index we're migrating from might
* contain documents created by a plugin that is disabled in the Kibana
* instance performing this migration. We merge the
* _meta.migrationMappingPropertyHashes mappings from the source index into
* the targetMappings to ensure that any `migrationPropertyHashes` for
* disabled plugins aren't lost.
* @remarks When another instance already completed a migration, the existing
* target index might contain documents and mappings created by a plugin that
* is disabled in the current Kibana instance performing this migration.
* Mapping updates are commutative (deeply merged) by Elasticsearch, except
* for the `_meta` key. By merging the `_meta.migrationMappingPropertyHashes`
* mappings from the existing target index index into the targetMappings we
* ensure that any `migrationPropertyHashes` for disabled plugins aren't lost.
*
* Right now we don't use these `migrationPropertyHashes` but it could be used
* in the future to detect if mappings were changed. If mappings weren't
Expand Down Expand Up @@ -209,7 +209,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// index
sourceIndex: Option.none,
targetIndex: `${stateP.indexPrefix}_${stateP.kibanaVersion}_001`,
targetIndexMappings: disableUnknownTypeMappingFields(
targetIndexMappings: mergeMigrationMappingPropertyHashes(
stateP.targetIndexMappings,
indices[aliases[stateP.currentAlias]].mappings
),
Expand Down Expand Up @@ -242,7 +242,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
controlState: 'SET_SOURCE_WRITE_BLOCK',
sourceIndex: Option.some(source) as Option.Some<string>,
targetIndex: target,
targetIndexMappings: mergeMigrationMappingPropertyHashes(
targetIndexMappings: disableUnknownTypeMappingFields(
stateP.targetIndexMappings,
indices[source].mappings
),
Expand Down Expand Up @@ -279,7 +279,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
controlState: 'LEGACY_SET_WRITE_BLOCK',
sourceIndex: Option.some(legacyReindexTarget) as Option.Some<string>,
targetIndex: target,
targetIndexMappings: mergeMigrationMappingPropertyHashes(
targetIndexMappings: disableUnknownTypeMappingFields(
stateP.targetIndexMappings,
indices[stateP.legacyIndex].mappings
),
Expand Down

0 comments on commit 9baf383

Please sign in to comment.