diff --git a/src/plugins/saved_objects_management/public/management_section/objects_table/components/table.tsx b/src/plugins/saved_objects_management/public/management_section/objects_table/components/table.tsx index 75f36e20eb084f..ed79135f1f09fb 100644 --- a/src/plugins/saved_objects_management/public/management_section/objects_table/components/table.tsx +++ b/src/plugins/saved_objects_management/public/management_section/objects_table/components/table.tsx @@ -74,6 +74,8 @@ interface TableState { activeAction?: SavedObjectsManagementAction; } +const MAX_PAGINATED_ITEM = 10000; + export class Table extends PureComponent { state: TableState = { isSearchTextValid: true, @@ -150,10 +152,12 @@ export class Table extends PureComponent { allowedTypes, } = this.props; + const cappedTotalItemCount = Math.min(totalItemCount, MAX_PAGINATED_ITEM); + const pagination = { pageIndex, pageSize, - totalItemCount, + totalItemCount: cappedTotalItemCount, pageSizeOptions: [5, 10, 20, 50], }; @@ -321,6 +325,7 @@ export class Table extends PureComponent { ); const activeActionContents = this.state.activeAction?.render() ?? null; + const exceededResultCount = totalItemCount > MAX_PAGINATED_ITEM; return ( @@ -392,6 +397,18 @@ export class Table extends PureComponent { /> {queryParseError} + {exceededResultCount && ( + <> + + + + + + )}
{ savedObjectsClient = savedObjectsClientMock.create(); }); - it('calls the saved object client with the correct parameters', async () => { + it('calls `client.createPointInTimeFinder` with the correct parameters', async () => { + const query: SavedObjectsFindOptions = { + type: ['some-type', 'another-type'], + }; + + savedObjectsClient.find.mockResolvedValue({ + saved_objects: [], + total: 1, + per_page: 20, + page: 1, + }); + + await findAll(savedObjectsClient, query); + + expect(savedObjectsClient.createPointInTimeFinder).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.createPointInTimeFinder).toHaveBeenCalledWith(query); + }); + + it('returns the results from the PIT search', async () => { const query: SavedObjectsFindOptions = { type: ['some-type', 'another-type'], }; @@ -41,45 +59,40 @@ describe('findAll', () => { const results = await findAll(savedObjectsClient, query); expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); - expect(savedObjectsClient.find).toHaveBeenCalledWith({ - ...query, - page: 1, - }); + expect(savedObjectsClient.find).toHaveBeenCalledWith( + expect.objectContaining({ + ...query, + }) + ); expect(results).toEqual([createObj(1), createObj(2)]); }); - it('recursively call find until all objects are fetched', async () => { + it('works when the PIT search returns multiple batches', async () => { const query: SavedObjectsFindOptions = { type: ['some-type', 'another-type'], + perPage: 2, }; const objPerPage = 2; - savedObjectsClient.find.mockImplementation(({ page }) => { - const firstInPage = (page! - 1) * objPerPage + 1; + let callCount = 0; + savedObjectsClient.find.mockImplementation(({}) => { + callCount++; + const firstInPage = (callCount - 1) * objPerPage + 1; return Promise.resolve({ - saved_objects: [createObj(firstInPage), createObj(firstInPage + 1)], + saved_objects: + callCount > 3 + ? [createObj(firstInPage)] + : [createObj(firstInPage), createObj(firstInPage + 1)], total: objPerPage * 3, per_page: objPerPage, - page: page!, + page: callCount!, }); }); const results = await findAll(savedObjectsClient, query); - expect(savedObjectsClient.find).toHaveBeenCalledTimes(3); - expect(savedObjectsClient.find).toHaveBeenCalledWith({ - ...query, - page: 1, - }); - expect(savedObjectsClient.find).toHaveBeenCalledWith({ - ...query, - page: 2, - }); - expect(savedObjectsClient.find).toHaveBeenCalledWith({ - ...query, - page: 3, - }); - expect(results).toEqual(times(6, (num) => createObj(num + 1))); + expect(savedObjectsClient.find).toHaveBeenCalledTimes(4); + expect(results).toEqual(times(7, (num) => createObj(num + 1))); }); }); diff --git a/src/plugins/saved_objects_management/server/lib/find_all.ts b/src/plugins/saved_objects_management/server/lib/find_all.ts index 08681758752be9..8d908234c69617 100644 --- a/src/plugins/saved_objects_management/server/lib/find_all.ts +++ b/src/plugins/saved_objects_management/server/lib/find_all.ts @@ -6,30 +6,20 @@ * Side Public License, v 1. */ -import { SavedObjectsClientContract, SavedObject, SavedObjectsFindOptions } from 'src/core/server'; +import { + SavedObjectsClientContract, + SavedObject, + SavedObjectsCreatePointInTimeFinderOptions, +} from 'src/core/server'; export const findAll = async ( client: SavedObjectsClientContract, - findOptions: SavedObjectsFindOptions + findOptions: SavedObjectsCreatePointInTimeFinderOptions ): Promise => { - return recursiveFind(client, findOptions, 1, []); -}; - -const recursiveFind = async ( - client: SavedObjectsClientContract, - findOptions: SavedObjectsFindOptions, - page: number, - allObjects: SavedObject[] -): Promise => { - const objects = await client.find({ - ...findOptions, - page, - }); - - allObjects.push(...objects.saved_objects); - if (allObjects.length < objects.total) { - return recursiveFind(client, findOptions, page + 1, allObjects); + const finder = client.createPointInTimeFinder(findOptions); + const results: SavedObject[] = []; + for await (const result of finder.find()) { + results.push(...result.saved_objects); } - - return allObjects; + return results; }; diff --git a/src/plugins/saved_objects_management/server/routes/index.test.ts b/src/plugins/saved_objects_management/server/routes/index.test.ts index 3ec6afe1c0bbc1..9c3d235a83e367 100644 --- a/src/plugins/saved_objects_management/server/routes/index.test.ts +++ b/src/plugins/saved_objects_management/server/routes/index.test.ts @@ -24,7 +24,7 @@ describe('registerRoutes', () => { expect(httpSetup.createRouter).toHaveBeenCalledTimes(1); expect(router.get).toHaveBeenCalledTimes(3); - expect(router.post).toHaveBeenCalledTimes(3); + expect(router.post).toHaveBeenCalledTimes(2); expect(router.get).toHaveBeenCalledWith( expect.objectContaining({ @@ -56,11 +56,5 @@ describe('registerRoutes', () => { }), expect.any(Function) ); - expect(router.post).toHaveBeenCalledWith( - expect.objectContaining({ - path: '/api/kibana/management/saved_objects/scroll/export', - }), - expect.any(Function) - ); }); }); diff --git a/src/plugins/saved_objects_management/server/routes/index.ts b/src/plugins/saved_objects_management/server/routes/index.ts index b5b461575604db..5370088d279772 100644 --- a/src/plugins/saved_objects_management/server/routes/index.ts +++ b/src/plugins/saved_objects_management/server/routes/index.ts @@ -11,7 +11,6 @@ import { ISavedObjectsManagement } from '../services'; import { registerFindRoute } from './find'; import { registerBulkGetRoute } from './bulk_get'; import { registerScrollForCountRoute } from './scroll_count'; -import { registerScrollForExportRoute } from './scroll_export'; import { registerRelationshipsRoute } from './relationships'; import { registerGetAllowedTypesRoute } from './get_allowed_types'; @@ -25,7 +24,6 @@ export function registerRoutes({ http, managementServicePromise }: RegisterRoute registerFindRoute(router, managementServicePromise); registerBulkGetRoute(router, managementServicePromise); registerScrollForCountRoute(router); - registerScrollForExportRoute(router); registerRelationshipsRoute(router, managementServicePromise); registerGetAllowedTypesRoute(router); } diff --git a/src/plugins/saved_objects_management/server/routes/scroll_count.ts b/src/plugins/saved_objects_management/server/routes/scroll_count.ts index 89a895adf60084..9a021be4c0cde8 100644 --- a/src/plugins/saved_objects_management/server/routes/scroll_count.ts +++ b/src/plugins/saved_objects_management/server/routes/scroll_count.ts @@ -7,7 +7,7 @@ */ import { schema } from '@kbn/config-schema'; -import { IRouter, SavedObjectsFindOptions } from 'src/core/server'; +import { IRouter, SavedObjectsCreatePointInTimeFinderOptions } from 'src/core/server'; import { chain } from 'lodash'; import { findAll } from '../lib'; @@ -42,7 +42,7 @@ export const registerScrollForCountRoute = (router: IRouter) => { .value(); const client = getClient({ includedHiddenTypes }); - const findOptions: SavedObjectsFindOptions = { + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: typesToInclude, perPage: 1000, }; diff --git a/src/plugins/saved_objects_management/server/routes/scroll_export.ts b/src/plugins/saved_objects_management/server/routes/scroll_export.ts deleted file mode 100644 index 8d11437af661b9..00000000000000 --- a/src/plugins/saved_objects_management/server/routes/scroll_export.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { schema } from '@kbn/config-schema'; -import { IRouter } from 'src/core/server'; -import { chain } from 'lodash'; -import { findAll } from '../lib'; - -export const registerScrollForExportRoute = (router: IRouter) => { - router.post( - { - path: '/api/kibana/management/saved_objects/scroll/export', - validate: { - body: schema.object({ - typesToInclude: schema.arrayOf(schema.string()), - }), - }, - }, - router.handleLegacyErrors(async (context, req, res) => { - const { typesToInclude } = req.body; - const { getClient, typeRegistry } = context.core.savedObjects; - const includedHiddenTypes = chain(typesToInclude) - .uniq() - .filter( - (type) => typeRegistry.isHidden(type) && typeRegistry.isImportableAndExportable(type) - ) - .value(); - - const client = getClient({ includedHiddenTypes }); - - const objects = await findAll(client, { - perPage: 1000, - type: typesToInclude, - }); - - return res.ok({ - body: objects.map((hit) => { - return { - _id: hit.id, - _source: hit.attributes, - _meta: { - savedObjectVersion: 2, - }, - _migrationVersion: hit.migrationVersion, - _references: hit.references || [], - }; - }), - }); - }) - ); -}; diff --git a/test/api_integration/apis/saved_objects_management/scroll_count.ts b/test/api_integration/apis/saved_objects_management/scroll_count.ts index 088b26d8205da4..ffb275e8656f05 100644 --- a/test/api_integration/apis/saved_objects_management/scroll_count.ts +++ b/test/api_integration/apis/saved_objects_management/scroll_count.ts @@ -18,77 +18,132 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); describe('scroll_count', () => { - before(async () => { - await esArchiver.load( - 'test/api_integration/fixtures/es_archiver/management/saved_objects/scroll_count' - ); - }); - after(async () => { - await esArchiver.unload( - 'test/api_integration/fixtures/es_archiver/management/saved_objects/scroll_count' - ); - }); + describe('with less than 10k objects', () => { + before(async () => { + await esArchiver.load( + 'test/api_integration/fixtures/es_archiver/management/saved_objects/scroll_count' + ); + }); + after(async () => { + await esArchiver.unload( + 'test/api_integration/fixtures/es_archiver/management/saved_objects/scroll_count' + ); + }); + + it('returns the count for each included types', async () => { + const res = await supertest + .post(apiUrl) + .send({ + typesToInclude: defaultTypes, + }) + .expect(200); - it('returns the count for each included types', async () => { - const res = await supertest - .post(apiUrl) - .send({ - typesToInclude: defaultTypes, - }) - .expect(200); - - expect(res.body).to.eql({ - dashboard: 2, - 'index-pattern': 1, - search: 1, - visualization: 2, + expect(res.body).to.eql({ + dashboard: 2, + 'index-pattern': 1, + search: 1, + visualization: 2, + }); }); - }); - it('only returns count for types to include', async () => { - const res = await supertest - .post(apiUrl) - .send({ - typesToInclude: ['dashboard', 'search'], - }) - .expect(200); - - expect(res.body).to.eql({ - dashboard: 2, - search: 1, + it('only returns count for types to include', async () => { + const res = await supertest + .post(apiUrl) + .send({ + typesToInclude: ['dashboard', 'search'], + }) + .expect(200); + + expect(res.body).to.eql({ + dashboard: 2, + search: 1, + }); }); - }); - it('filters on title when `searchString` is provided', async () => { - const res = await supertest - .post(apiUrl) - .send({ - typesToInclude: defaultTypes, - searchString: 'Amazing', - }) - .expect(200); - - expect(res.body).to.eql({ - dashboard: 1, - visualization: 1, - 'index-pattern': 0, - search: 0, + it('filters on title when `searchString` is provided', async () => { + const res = await supertest + .post(apiUrl) + .send({ + typesToInclude: defaultTypes, + searchString: 'Amazing', + }) + .expect(200); + + expect(res.body).to.eql({ + dashboard: 1, + visualization: 1, + 'index-pattern': 0, + search: 0, + }); + }); + + it('includes all requested types even when none match the search', async () => { + const res = await supertest + .post(apiUrl) + .send({ + typesToInclude: ['dashboard', 'search', 'visualization'], + searchString: 'nothing-will-match', + }) + .expect(200); + + expect(res.body).to.eql({ + dashboard: 0, + visualization: 0, + search: 0, + }); }); }); - it('includes all requested types even when none match the search', async () => { - const res = await supertest - .post(apiUrl) - .send({ - typesToInclude: ['dashboard', 'search', 'visualization'], - searchString: 'nothing-will-match', - }) - .expect(200); - - expect(res.body).to.eql({ - dashboard: 0, - visualization: 0, - search: 0, + describe('scroll_count with more than 10k objects', () => { + const importVisualizations = async ({ + startIdx = 1, + endIdx, + }: { + startIdx?: number; + endIdx: number; + }) => { + const fileChunks: string[] = []; + for (let i = startIdx; i <= endIdx; i++) { + const id = `test-vis-${i}`; + fileChunks.push( + JSON.stringify({ + type: 'visualization', + id, + attributes: { + title: `My visualization (${i})`, + uiStateJSON: '{}', + visState: '{}', + }, + references: [], + }) + ); + } + + await supertest + .post(`/api/saved_objects/_import`) + .attach('file', Buffer.from(fileChunks.join('\n'), 'utf8'), 'export.ndjson') + .expect(200); + }; + + before(async () => { + await importVisualizations({ startIdx: 1, endIdx: 6000 }); + await importVisualizations({ startIdx: 6001, endIdx: 12000 }); + }); + after(async () => { + await esArchiver.emptyKibanaIndex(); + }); + + it('returns the correct count for each included types', async () => { + const res = await supertest + .post(apiUrl) + .send({ + typesToInclude: ['visualization'], + }) + .expect(200); + + expect(res.body).to.eql({ + visualization: 12000, + }); }); }); }); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 9594ff842822b0..c816c10547a146 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -4284,7 +4284,6 @@ "savedObjectsManagement.objectsTable.deleteSavedObjectsConfirmModal.titleColumnName": "タイトル", "savedObjectsManagement.objectsTable.deleteSavedObjectsConfirmModal.typeColumnName": "型", "savedObjectsManagement.objectsTable.deleteSavedObjectsConfirmModalTitle": "保存されたオブジェクトの削除", - "savedObjectsManagement.objectsTable.export.dangerNotification": "エクスポートを生成できません", "savedObjectsManagement.objectsTable.export.successNotification": "ファイルはバックグラウンドでダウンロード中です", "savedObjectsManagement.objectsTable.export.successWithExcludedObjectsNotification": "ファイルはバックグラウンドでダウンロード中です。一部のオブジェクトはエクスポートから除外されました。除外されたオブジェクトの一覧は、エクスポートされたファイルの最後の行をご覧ください。", "savedObjectsManagement.objectsTable.export.successWithMissingRefsNotification": "ファイルはバックグラウンドでダウンロード中です。一部の関連オブジェクトが見つかりませんでした。足りないオブジェクトの一覧は、エクスポートされたファイルの最後の行をご覧ください。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 47339d3b376fe3..fc6f651fe3f245 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -4325,7 +4325,6 @@ "savedObjectsManagement.objectsTable.deleteSavedObjectsConfirmModal.titleColumnName": "标题", "savedObjectsManagement.objectsTable.deleteSavedObjectsConfirmModal.typeColumnName": "类型", "savedObjectsManagement.objectsTable.deleteSavedObjectsConfirmModalTitle": "删除已保存对象", - "savedObjectsManagement.objectsTable.export.dangerNotification": "无法生成导出", "savedObjectsManagement.objectsTable.export.successNotification": "您的文件正在后台下载", "savedObjectsManagement.objectsTable.export.successWithExcludedObjectsNotification": "您的文件正在后台下载。一些对象已从导出中排除。有关已排除对象列表，请查看导出文件的最后一行。", "savedObjectsManagement.objectsTable.export.successWithMissingRefsNotification": "您的文件正在后台下载。找不到某些相关对象。有关缺失对象列表，请查看导出文件的最后一行。",