From cc13ed43f1429aed5189a6a94c08e0c15b31da7b Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Thu, 13 Apr 2023 06:02:56 +0200 Subject: [PATCH] Update ES archiver to support multiple SO indices --- .../core-saved-objects-server/index.ts | 6 ++- .../src/saved_objects_index_pattern.ts | 9 ++++ .../src/actions/empty_kibana_index.ts | 3 +- packages/kbn-es-archiver/src/actions/load.ts | 10 +++-- .../lib/docs/generate_doc_records_stream.ts | 5 ++- .../src/lib/indices/create_index_stream.ts | 8 +++- .../src/lib/indices/delete_index_stream.ts | 3 +- .../indices/generate_index_records_stream.ts | 6 ++- .../src/lib/indices/kibana_index.ts | 45 ++++++++++++------- packages/kbn-es-archiver/tsconfig.json | 1 + .../kibana_server/extend_es_archiver.ts | 4 +- .../tsconfig.json | 1 + 12 files changed, 72 insertions(+), 29 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index ed4981770f281e..e24edb73f4d593 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -52,7 +52,11 @@ export type { SavedObjectsExportablePredicate, } from './src/saved_objects_management'; export type { SavedObjectStatusMeta } from './src/saved_objects_status'; -export { MAIN_SAVED_OBJECT_INDEX } from './src/saved_objects_index_pattern'; +export { + MAIN_SAVED_OBJECT_INDEX, + TASK_MANAGER_SAVED_OBJECT_INDEX, + SavedObjectsIndexPatterns, +} from './src/saved_objects_index_pattern'; export type { SavedObjectsType, SavedObjectTypeExcludeFromUpgradeFilterHook, diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts index 53d05d7c1d45f9..b461f5e975915d 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts @@ -6,4 +6,13 @@ * Side Public License, v 1. */ +/** + * Collect and centralize the names of the different saved object indices. + * Note that all of them start with the '.kibana' prefix. + * There are multiple places in the code that these indices have the form .kibana*. + * However, beware that there are some system indices that have the same prefix + * but are NOT used to store saved objects, e.g.: .kibana_security_session_1 + */ export const MAIN_SAVED_OBJECT_INDEX = '.kibana'; +export const TASK_MANAGER_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_task_manager`; +export const SavedObjectsIndexPatterns = [MAIN_SAVED_OBJECT_INDEX, TASK_MANAGER_SAVED_OBJECT_INDEX]; diff --git a/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts index c9b67e4745d45a..cfc809636fd36c 100644 --- a/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts +++ b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts @@ -10,6 +10,7 @@ import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { KbnClient } from '@kbn/test'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib'; export async function emptyKibanaIndexAction({ @@ -25,6 +26,6 @@ export async function emptyKibanaIndexAction({ await cleanKibanaIndices({ client, stats, log }); await migrateKibanaIndex(kbnClient); - stats.createdIndex('.kibana'); + SavedObjectsIndexPatterns.forEach((indexPattern) => stats.createdIndex(indexPattern)); return stats.toJSON(); } diff --git a/packages/kbn-es-archiver/src/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts index b6a4c46d42743c..efda58193b118d 100644 --- a/packages/kbn-es-archiver/src/actions/load.ts +++ b/packages/kbn-es-archiver/src/actions/load.ts @@ -14,6 +14,7 @@ import { REPO_ROOT } from '@kbn/repo-info'; import type { KbnClient } from '@kbn/test'; import type { Client } from '@elastic/elasticsearch'; import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { ES_CLIENT_HEADERS } from '../client_headers'; import { @@ -104,14 +105,15 @@ export async function loadAction({ } ); - // If we affected the Kibana index, we need to ensure it's migrated... - if (Object.keys(result).some((k) => k.startsWith('.kibana'))) { + // If we affected saved objects indices, we need to ensure they are migrated... + if (Object.keys(result).some((k) => k.startsWith(MAIN_SAVED_OBJECT_INDEX))) { await migrateKibanaIndex(kbnClient); log.debug('[%s] Migrated Kibana index after loading Kibana data', name); if (kibanaPluginIds.includes('spaces')) { - await createDefaultSpace({ client, index: '.kibana' }); - log.debug('[%s] Ensured that default space exists in .kibana', name); + // WARNING affected by #104081. Assumes 'spaces' saved objects are stored in MAIN_SAVED_OBJECT_INDEX + await createDefaultSpace({ client, index: MAIN_SAVED_OBJECT_INDEX }); + log.debug(`[%s] Ensured that default space exists in ${MAIN_SAVED_OBJECT_INDEX}`, name); } } diff --git a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts index 6e3310a7347e73..f0c6db9c89fcbf 100644 --- a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts @@ -8,6 +8,7 @@ import { Transform } from 'stream'; import type { Client } from '@elastic/elasticsearch'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { Progress } from '../progress'; import { ES_CLIENT_HEADERS } from '../../client_headers'; @@ -78,7 +79,9 @@ export function createGenerateDocRecordsStream({ // if keepIndexNames is false, rewrite the .kibana_* index to .kibana_1 so that // when it is loaded it can skip migration, if possible index: - hit._index.startsWith('.kibana') && !keepIndexNames ? '.kibana_1' : hit._index, + hit._index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !keepIndexNames + ? `${MAIN_SAVED_OBJECT_INDEX}_1` + : hit._index, data_stream: dataStream, id: hit._id, source: hit._source, diff --git a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts index 38f4bed755262d..a13c1b7e856e9c 100644 --- a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts @@ -14,6 +14,10 @@ import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + MAIN_SAVED_OBJECT_INDEX, + TASK_MANAGER_SAVED_OBJECT_INDEX, +} from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { deleteKibanaIndices } from './kibana_index'; import { deleteIndex } from './delete_index'; @@ -96,8 +100,8 @@ export function createCreateIndexStream({ async function handleIndex(record: DocRecord) { const { index, settings, mappings, aliases } = record.value; - const isKibanaTaskManager = index.startsWith('.kibana_task_manager'); - const isKibana = index.startsWith('.kibana') && !isKibanaTaskManager; + const isKibanaTaskManager = index.startsWith(TASK_MANAGER_SAVED_OBJECT_INDEX); + const isKibana = index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !isKibanaTaskManager; if (docsOnly) { return; diff --git a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts index c7633465ccc4cb..1c10ab0fc273c9 100644 --- a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts @@ -10,6 +10,7 @@ import { Transform } from 'stream'; import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; import { cleanKibanaIndices } from './kibana_index'; @@ -28,7 +29,7 @@ export function createDeleteIndexStream(client: Client, stats: Stats, log: Tooli if (record.type === 'index') { const { index } = record.value; - if (index.startsWith('.kibana')) { + if (index.startsWith(MAIN_SAVED_OBJECT_INDEX)) { await cleanKibanaIndices({ client, stats, log }); } else { await deleteIndex({ client, stats, log, index }); diff --git a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts index de32e93e273989..2f2dd60982a940 100644 --- a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts @@ -9,6 +9,7 @@ import type { Client } from '@elastic/elasticsearch'; import { Transform } from 'stream'; import { ToolingLog } from '@kbn/tooling-log'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { ES_CLIENT_HEADERS } from '../../client_headers'; import { getIndexTemplate } from '..'; @@ -100,7 +101,10 @@ export function createGenerateIndexRecordsStream({ value: { // if keepIndexNames is false, rewrite the .kibana_* index to .kibana_1 so that // when it is loaded it can skip migration, if possible - index: index.startsWith('.kibana') && !keepIndexNames ? '.kibana_1' : index, + index: + index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !keepIndexNames + ? `${MAIN_SAVED_OBJECT_INDEX}_1` + : index, settings, mappings, aliases, diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index 6a02113bbf733a..75aa5f5fce3d5e 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -11,12 +11,17 @@ import { inspect } from 'util'; import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { KbnClient } from '@kbn/test'; +import { + MAIN_SAVED_OBJECT_INDEX, + SavedObjectsIndexPatterns, + TASK_MANAGER_SAVED_OBJECT_INDEX, +} from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; import { ES_CLIENT_HEADERS } from '../../client_headers'; /** - * Deletes all indices that start with `.kibana`, or if onlyTaskManager==true, all indices that start with `.kibana_task_manager` + * Deletes all saved object indices, or if onlyTaskManager==true, it deletes task_manager indices */ export async function deleteKibanaIndices({ client, @@ -29,8 +34,10 @@ export async function deleteKibanaIndices({ onlyTaskManager?: boolean; log: ToolingLog; }) { - const indexPattern = onlyTaskManager ? '.kibana_task_manager*' : '.kibana*'; - const indexNames = await fetchKibanaIndices(client, indexPattern); + // WARNING note that we are deleting ALL .kibana* indices here, NOT only the saved object ones + const indexNames = (await fetchKibanaIndices(client)).filter( + (indexName) => !onlyTaskManager || indexName.includes(TASK_MANAGER_SAVED_OBJECT_INDEX) + ); if (!indexNames.length) { return; } @@ -65,22 +72,28 @@ export async function migrateKibanaIndex(kbnClient: KbnClient) { } /** - * Migrations mean that the Kibana index will look something like: - * .kibana, .kibana_1, .kibana_323, etc. This finds all indices starting - * with .kibana, then filters out any that aren't actually Kibana's core - * index (e.g. we don't want to remove .kibana_task_manager or the like). + * Check if the given index is a Kibana saved object index. + * This includes most .kibana_* + * but we must make sure that indices such as '.kibana_security_session_1' are NOT deleted. + * + * IMPORTANT + * Note that we can have more than 2 system indices (different SO types can go to different indices) + * ATM we have '.kibana', '.kibana_task_manager', '.kibana_cases' + * This method also takes into account legacy indices: .kibana_1, .kibana_task_manager_1. + * @param [index] the name of the index to check + * @returns boolean 'true' if the index is a Kibana saved object index. */ + +const LEGACY_INDICES_REGEXP = new RegExp(`^(${SavedObjectsIndexPatterns.join('|')})(:?_\\d*)?$`); +const INDICES_REGEXP = new RegExp(`^(${SavedObjectsIndexPatterns.join('|')})_(pre)?\\d+.\\d+.\\d+`); + function isKibanaIndex(index?: string): index is string { - return Boolean( - index && - (/^\.kibana(:?_\d*)?$/.test(index) || - /^\.kibana(_task_manager)?_(pre)?\d+\.\d+\.\d+/.test(index)) - ); + return Boolean(index && (LEGACY_INDICES_REGEXP.test(index) || INDICES_REGEXP.test(index))); } -async function fetchKibanaIndices(client: Client, indexPattern: string) { +async function fetchKibanaIndices(client: Client) { const resp = await client.cat.indices( - { index: indexPattern, format: 'json' }, + { index: `${MAIN_SAVED_OBJECT_INDEX}*`, format: 'json' }, { headers: ES_CLIENT_HEADERS, } @@ -107,7 +120,7 @@ export async function cleanKibanaIndices({ while (true) { const resp = await client.deleteByQuery( { - index: `.kibana,.kibana_task_manager`, + index: SavedObjectsIndexPatterns, body: { query: { bool: { @@ -144,7 +157,7 @@ export async function cleanKibanaIndices({ `.kibana rather than deleting the whole index` ); - stats.deletedIndex('.kibana'); + SavedObjectsIndexPatterns.forEach((indexPattern) => stats.deletedIndex(indexPattern)); } export async function createDefaultSpace({ index, client }: { index: string; client: Client }) { diff --git a/packages/kbn-es-archiver/tsconfig.json b/packages/kbn-es-archiver/tsconfig.json index 0301480548fc74..15fccdf68be4f1 100644 --- a/packages/kbn-es-archiver/tsconfig.json +++ b/packages/kbn-es-archiver/tsconfig.json @@ -11,6 +11,7 @@ "**/*.ts" ], "kbn_references": [ + "@kbn/core-saved-objects-server", "@kbn/dev-utils", "@kbn/test", "@kbn/tooling-log", diff --git a/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts b/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts index 98c28960bf5230..4c2613d273c4a2 100644 --- a/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts +++ b/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import type { ProvidedType } from '@kbn/test'; import type { EsArchiverProvider } from '../es_archiver'; @@ -13,7 +14,6 @@ import type { RetryService } from '../retry'; import type { KibanaServerProvider } from './kibana_server'; const ES_ARCHIVER_LOAD_METHODS = ['load', 'loadIfNeeded', 'unload', 'emptyKibanaIndex'] as const; -const KIBANA_INDEX = '.kibana'; interface Options { esArchiver: ProvidedType; @@ -38,7 +38,7 @@ export function extendEsArchiver({ esArchiver, kibanaServer, retry, defaults }: const statsKeys = Object.keys(stats); const kibanaKeys = statsKeys.filter( // this also matches stats keys like '.kibana_1' and '.kibana_2,.kibana_1' - (key) => key.includes(KIBANA_INDEX) && stats[key].created + (key) => key.includes(MAIN_SAVED_OBJECT_INDEX) && stats[key].created ); // if the kibana index was created by the esArchiver then update the uiSettings diff --git a/packages/kbn-ftr-common-functional-services/tsconfig.json b/packages/kbn-ftr-common-functional-services/tsconfig.json index 639991bb2ce77c..3641c807e4d6d3 100644 --- a/packages/kbn-ftr-common-functional-services/tsconfig.json +++ b/packages/kbn-ftr-common-functional-services/tsconfig.json @@ -11,6 +11,7 @@ "**/*.ts", ], "kbn_references": [ + "@kbn/core-saved-objects-server", "@kbn/tooling-log", "@kbn/es-archiver", "@kbn/test"