From 98e8a41f3a929f0530fa1449ae847dc4d1b206fb Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Mon, 30 Jan 2023 17:05:53 +0100 Subject: [PATCH] [FTR] `KbnClientSavedObjects` improvements (#149582) ## Summary Follow-up of https://github.com/elastic/kibana/pull/149188 - Use the bulkDelete API for `KbnClientSavedObjects.bulkDelete` - Create a dedicated `/_clean` endpoint for `KbnClientSavedObjects.clean` and `KbnClientSavedObjects.cleanStandardList` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../kbn_client/import_export/parse_archive.ts | 14 +- .../kbn_client/kbn_client_import_export.ts | 3 +- .../kbn_client/kbn_client_saved_objects.ts | 181 +++++++----------- packages/kbn-test/tsconfig.json | 1 + .../server/routes/kbn_client_so/clean.ts | 48 +++++ .../server/routes/kbn_client_so/index.ts | 2 + .../security_and_spaces/apis/clean.ts | 92 +++++++++ .../security_and_spaces/apis/index.ts | 1 + 8 files changed, 231 insertions(+), 111 deletions(-) create mode 100644 src/plugins/ftr_apis/server/routes/kbn_client_so/clean.ts create mode 100644 x-pack/test/ftr_apis/security_and_spaces/apis/clean.ts diff --git a/packages/kbn-test/src/kbn_client/import_export/parse_archive.ts b/packages/kbn-test/src/kbn_client/import_export/parse_archive.ts index b6b85ba521525..eda37b62cf0d6 100644 --- a/packages/kbn-test/src/kbn_client/import_export/parse_archive.ts +++ b/packages/kbn-test/src/kbn_client/import_export/parse_archive.ts @@ -14,9 +14,19 @@ export interface SavedObject { [key: string]: unknown; } -export async function parseArchive(path: string): Promise { +export async function parseArchive( + path: string, + { stripSummary = false }: { stripSummary?: boolean } = {} +): Promise { return (await Fs.readFile(path, 'utf-8')) .split(/\r?\n\r?\n/) .filter((line) => !!line) - .map((line) => JSON.parse(line)); + .map((line) => JSON.parse(line)) + .filter( + stripSummary + ? (object) => { + return object.type && object.id; + } + : () => true + ); } diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index 5259b50cfce85..f9646410f64b5 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -25,6 +25,7 @@ interface ImportApiResponse { success: boolean; [key: string]: unknown; } + export class KbnClientImportExport { constructor( public readonly log: ToolingLog, @@ -92,7 +93,7 @@ export class KbnClientImportExport { const src = this.resolveAndValidatePath(path); this.log.debug('unloading docs from archive at', src); - const objects = await parseArchive(src); + const objects = await parseArchive(src, { stripSummary: true }); this.log.info('deleting', objects.length, 'objects', { space: options?.space }); const { deleted, missing } = await this.savedObjects.bulkDelete({ diff --git a/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts b/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts index b896aef3d2f56..879050fd40dec 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts @@ -6,12 +6,9 @@ * Side Public License, v 1. */ -import { inspect } from 'util'; -import * as Rx from 'rxjs'; -import { mergeMap } from 'rxjs/operators'; -import { isAxiosResponseError } from '@kbn/dev-utils'; -import { createFailError } from '@kbn/dev-cli-errors'; -import { ToolingLog } from '@kbn/tooling-log'; +import { chunk } from 'lodash'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { SavedObjectsBulkDeleteResponse } from '@kbn/core-saved-objects-api-server'; import { KbnClientRequester, uriencode } from './kbn_client_requester'; @@ -57,22 +54,15 @@ interface MigrateResponse { result: Array<{ status: string }>; } -interface FindApiResponse { - saved_objects: Array<{ - type: string; - id: string; - [key: string]: unknown; - }>; - total: number; - per_page: number; - page: number; -} - interface CleanOptions { space?: string; types: string[]; } +interface CleanApiResponse { + deleted: number; +} + interface DeleteObjectsOptions { space?: string; objects: Array<{ @@ -81,13 +71,43 @@ interface DeleteObjectsOptions { }>; } -async function concurrently(maxConcurrency: number, arr: T[], fn: (item: T) => Promise) { - if (arr.length) { - await Rx.lastValueFrom( - Rx.from(arr).pipe(mergeMap(async (item) => await fn(item), maxConcurrency)) - ); - } -} +const DELETE_CHUNK_SIZE = 50; + +// add types here +const STANDARD_LIST_TYPES = [ + 'url', + 'index-pattern', + 'action', + 'query', + 'alert', + 'graph-workspace', + 'tag', + 'visualization', + 'canvas-element', + 'canvas-workpad', + 'dashboard', + 'search', + 'lens', + 'map', + 'cases', + 'uptime-dynamic-settings', + 'osquery-saved-query', + 'osquery-pack', + 'infrastructure-ui-source', + 'metrics-explorer-view', + 'inventory-view', + 'infrastructure-monitoring-log-view', + 'apm-indices', + // Fleet saved object types + 'ingest-outputs', + 'ingest-download-sources', + 'ingest-agent-policies', + 'ingest-package-policies', + 'epm-packages', + 'epm-packages-assets', + 'fleet-preconfiguration-deletion-record', + 'fleet-fleet-server-host', +]; /** * SO client for FTR. @@ -194,74 +214,22 @@ export class KbnClientSavedObjects { public async clean(options: CleanOptions) { this.log.debug('Cleaning all saved objects', { space: options.space }); - let deleted = 0; - - while (true) { - const resp = await this.requester.request({ - method: 'GET', - path: options.space - ? uriencode`/s/${options.space}/internal/ftr/kbn_client_so/_find` - : `/internal/ftr/kbn_client_so/_find`, - query: { - per_page: 1000, - type: options.types, - fields: 'none', - }, - }); - - this.log.info('deleting batch of', resp.data.saved_objects.length, 'objects'); - const deletion = await this.bulkDelete({ - space: options.space, - objects: resp.data.saved_objects, - }); - deleted += deletion.deleted; - - if (resp.data.total <= resp.data.per_page) { - break; - } - } + const resp = await this.requester.request({ + method: 'POST', + path: options.space + ? uriencode`/s/${options.space}/internal/ftr/kbn_client_so/_clean` + : `/internal/ftr/kbn_client_so/_clean`, + body: { + types: options.types, + }, + }); + const deleted = resp.data.deleted; this.log.success('deleted', deleted, 'objects'); } public async cleanStandardList(options?: { space?: string }) { - // add types here - const types = [ - 'url', - 'index-pattern', - 'action', - 'query', - 'alert', - 'graph-workspace', - 'tag', - 'visualization', - 'canvas-element', - 'canvas-workpad', - 'dashboard', - 'search', - 'lens', - 'map', - 'cases', - 'uptime-dynamic-settings', - 'osquery-saved-query', - 'osquery-pack', - 'infrastructure-ui-source', - 'metrics-explorer-view', - 'inventory-view', - 'infrastructure-monitoring-log-view', - 'apm-indices', - // Fleet saved object types - 'ingest-outputs', - 'ingest-download-sources', - 'ingest-agent-policies', - 'ingest-package-policies', - 'epm-packages', - 'epm-packages-assets', - 'fleet-preconfiguration-deletion-record', - 'fleet-fleet-server-host', - ]; - - const newOptions = { types, space: options?.space }; + const newOptions = { types: STANDARD_LIST_TYPES, space: options?.space }; await this.clean(newOptions); } @@ -269,28 +237,25 @@ export class KbnClientSavedObjects { let deleted = 0; let missing = 0; - await concurrently(20, options.objects, async (obj) => { - try { - await this.requester.request({ - method: 'DELETE', - path: options.space - ? uriencode`/s/${options.space}/internal/ftr/kbn_client_so/${obj.type}/${obj.id}` - : uriencode`/internal/ftr/kbn_client_so/${obj.type}/${obj.id}`, - }); - deleted++; - } catch (error) { - if (isAxiosResponseError(error)) { - if (error.response.status === 404) { - missing++; - return; - } - - throw createFailError(`${error.response.status} resp: ${inspect(error.response.data)}`); - } + const chunks = chunk(options.objects, DELETE_CHUNK_SIZE); - throw error; - } - }); + for (let i = 0; i < chunks.length; i++) { + const objects = chunks[i]; + const { data: response } = await this.requester.request({ + method: 'POST', + path: options.space + ? uriencode`/s/${options.space}/internal/ftr/kbn_client_so/_bulk_delete` + : uriencode`/internal/ftr/kbn_client_so/_bulk_delete`, + body: objects.map(({ type, id }) => ({ type, id })), + }); + response.statuses.forEach((status) => { + if (status.success) { + deleted++; + } else if (status.error?.statusCode === 404) { + missing++; + } + }); + } return { deleted, missing }; } diff --git a/packages/kbn-test/tsconfig.json b/packages/kbn-test/tsconfig.json index df6a698fb2e71..7d73db67a0b92 100644 --- a/packages/kbn-test/tsconfig.json +++ b/packages/kbn-test/tsconfig.json @@ -31,5 +31,6 @@ "@kbn/stdio-dev-helpers", "@kbn/babel-register", "@kbn/repo-packages", + "@kbn/core-saved-objects-api-server", ] } diff --git a/src/plugins/ftr_apis/server/routes/kbn_client_so/clean.ts b/src/plugins/ftr_apis/server/routes/kbn_client_so/clean.ts new file mode 100644 index 0000000000000..e6b485526bad8 --- /dev/null +++ b/src/plugins/ftr_apis/server/routes/kbn_client_so/clean.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IRouter } from '@kbn/core/server'; +import { schema } from '@kbn/config-schema'; +import { KBN_CLIENT_API_PREFIX, listHiddenTypes, catchAndReturnBoomErrors } from './utils'; + +export const registerCleanRoute = (router: IRouter) => { + router.post( + { + path: `${KBN_CLIENT_API_PREFIX}/_clean`, + options: { + tags: ['access:ftrApis'], + }, + validate: { + body: schema.object({ + types: schema.arrayOf(schema.string()), + }), + }, + }, + catchAndReturnBoomErrors(async (ctx, req, res) => { + const { types } = req.body; + const { savedObjects } = await ctx.core; + const hiddenTypes = listHiddenTypes(savedObjects.typeRegistry); + const soClient = savedObjects.getClient({ includedHiddenTypes: hiddenTypes }); + + const finder = soClient.createPointInTimeFinder({ type: types, perPage: 100 }); + let deleted = 0; + + for await (const response of finder.find()) { + const objects = response.saved_objects.map(({ type, id }) => ({ type, id })); + const { statuses } = await soClient.bulkDelete(objects, { force: true }); + deleted += statuses.filter((status) => status.success).length; + } + + return res.ok({ + body: { + deleted, + }, + }); + }) + ); +}; diff --git a/src/plugins/ftr_apis/server/routes/kbn_client_so/index.ts b/src/plugins/ftr_apis/server/routes/kbn_client_so/index.ts index b22f234d8455b..33975dbaa4551 100644 --- a/src/plugins/ftr_apis/server/routes/kbn_client_so/index.ts +++ b/src/plugins/ftr_apis/server/routes/kbn_client_so/index.ts @@ -13,6 +13,7 @@ import { registerDeleteRoute } from './delete'; import { registerFindRoute } from './find'; import { registerGetRoute } from './get'; import { registerUpdateRoute } from './update'; +import { registerCleanRoute } from './clean'; export const registerKbnClientSoRoutes = (router: IRouter) => { registerBulkDeleteRoute(router); @@ -21,4 +22,5 @@ export const registerKbnClientSoRoutes = (router: IRouter) => { registerFindRoute(router); registerGetRoute(router); registerUpdateRoute(router); + registerCleanRoute(router); }; diff --git a/x-pack/test/ftr_apis/security_and_spaces/apis/clean.ts b/x-pack/test/ftr_apis/security_and_spaces/apis/clean.ts new file mode 100644 index 0000000000000..dc3bacf5b35f4 --- /dev/null +++ b/x-pack/test/ftr_apis/security_and_spaces/apis/clean.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { USERS, User, ExpectedResponse } from '../../common/lib'; +import { FtrProviderContext } from '../services'; +import { createData, createTestSpaces, deleteData, deleteTestSpaces } from './test_utils'; + +// eslint-disable-next-line import/no-default-export +export default function (ftrContext: FtrProviderContext) { + const supertest = ftrContext.getService('supertestWithoutAuth'); + + describe('POST /internal/ftr/kbn_client_so/_clean', () => { + before(async () => { + await createTestSpaces(ftrContext); + }); + + after(async () => { + await deleteTestSpaces(ftrContext); + }); + + beforeEach(async () => { + await createData(ftrContext); + }); + + afterEach(async () => { + await deleteData(ftrContext); + }); + + const responses: Record = { + authorized: { + httpCode: 200, + expectResponse: ({ body }) => { + expect(body.deleted).to.be.greaterThan(0); + }, + }, + unauthorized: { + httpCode: 403, + expectResponse: ({ body }) => { + expect(body).to.eql({ + error: 'Forbidden', + message: 'Forbidden', + statusCode: 403, + }); + }, + }, + }; + const expectedResults: Record = { + authorized: [USERS.SUPERUSER], + unauthorized: [ + USERS.DEFAULT_SPACE_READ_USER, + USERS.DEFAULT_SPACE_SO_MANAGEMENT_WRITE_USER, + USERS.DEFAULT_SPACE_SO_TAGGING_READ_USER, + USERS.DEFAULT_SPACE_SO_TAGGING_WRITE_USER, + USERS.DEFAULT_SPACE_DASHBOARD_READ_USER, + USERS.DEFAULT_SPACE_VISUALIZE_READ_USER, + USERS.DEFAULT_SPACE_MAPS_READ_USER, + USERS.DEFAULT_SPACE_ADVANCED_SETTINGS_READ_USER, + USERS.NOT_A_KIBANA_USER, + ], + }; + + const createUserTest = ( + { username, password, description }: User, + { httpCode, expectResponse }: ExpectedResponse + ) => { + it(`returns expected ${httpCode} response for ${description ?? username}`, async () => { + await supertest + .post(`/internal/ftr/kbn_client_so/_clean`) + .send({ types: ['tag', 'dashboard', 'visualization'] }) + .auth(username, password) + .expect(httpCode) + .then(expectResponse); + }); + }; + + const createTestSuite = () => { + Object.entries(expectedResults).forEach(([responseId, users]) => { + const response: ExpectedResponse = responses[responseId]; + users.forEach((user) => { + createUserTest(user, response); + }); + }); + }; + + createTestSuite(); + }); +} diff --git a/x-pack/test/ftr_apis/security_and_spaces/apis/index.ts b/x-pack/test/ftr_apis/security_and_spaces/apis/index.ts index ca5bf0cb89221..e683baf38ba66 100644 --- a/x-pack/test/ftr_apis/security_and_spaces/apis/index.ts +++ b/x-pack/test/ftr_apis/security_and_spaces/apis/index.ts @@ -21,5 +21,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./delete')); loadTestFile(require.resolve('./find')); loadTestFile(require.resolve('./bulk_delete')); + loadTestFile(require.resolve('./clean')); }); }