Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[dot-kibana-split] Generalise FTR and integration tests to support multiple SO indices #154884

Merged
merged 2 commits into from
Apr 18, 2023
+942 −42
Merged

[dot-kibana-split] Generalise FTR and integration tests to support multiple SO indices #154884

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
984561b
Generalise FTR and integration tests to support multiple SO indices
gsoldevila Apr 13, 2023
4a2b57f
Merge branch 'dot-kibana-split' into dot-kibana-split-tests
gsoldevila Apr 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion packages/core/saved-objects/core-saved-objects-server/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ export type {
SavedObjectsExportablePredicate,
} from './src/saved_objects_management';
export type { SavedObjectStatusMeta } from './src/saved_objects_status';
export { MAIN_SAVED_OBJECT_INDEX } from './src/saved_objects_index_pattern';
export {
MAIN_SAVED_OBJECT_INDEX,
TASK_MANAGER_SAVED_OBJECT_INDEX,
SavedObjectsIndexPatterns,
} from './src/saved_objects_index_pattern';
export type {
SavedObjectsType,
SavedObjectTypeExcludeFromUpgradeFilterHook,
Expand Down
9 changes: 9 additions & 0 deletions packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,13 @@
* Side Public License, v 1.
*/

/**
* Collect and centralize the names of the different saved object indices.
* Note that all of them start with the '.kibana' prefix.
* There are multiple places in the code that these indices have the form .kibana*.
* However, beware that there are some system indices that have the same prefix
* but are NOT used to store saved objects, e.g.: .kibana_security_session_1
*/
export const MAIN_SAVED_OBJECT_INDEX = '.kibana';
export const TASK_MANAGER_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_task_manager`;
export const SavedObjectsIndexPatterns = [MAIN_SAVED_OBJECT_INDEX, TASK_MANAGER_SAVED_OBJECT_INDEX];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

2 NITS:

  1. case inconsistency between the individual indices constants and the list one
  2. not a big fan of the SavedObjectsIndexPatterns naming. I think something like ALL_SAVED_OBJECT_INDICES would be more explicit

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can do this as a follow up, to avoid merge conflict nightmares with the rest of "feature" PRs.

9 changes: 2 additions & 7 deletions src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@

import path from 'path';
import { unlink } from 'fs/promises';
import { REPO_ROOT } from '@kbn/repo-info';
import { Env } from '@kbn/config';
import { getEnvOptions } from '@kbn/config-mocks';
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import type { InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
import { Root } from '@kbn/core-root-server-internal';
Expand All @@ -19,8 +16,8 @@ import {
createRootWithCorePlugins,
type TestElasticsearchUtils,
} from '@kbn/core-test-helpers-kbn-server';
import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';

const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version;
const logFilePath = path.join(__dirname, '7.7.2_xpack_100k.log');

async function removeLogFile() {
Expand Down Expand Up @@ -105,8 +102,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {
await new Promise((resolve) => setTimeout(resolve, 10000));
};

const migratedIndex = `.kibana_${kibanaVersion}_001`;

beforeAll(async () => {
await removeLogFile();
await startServers({
Expand All @@ -121,7 +116,7 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {

it('copies all the document of the previous index to the new one', async () => {
const migratedIndexResponse = await esClient.count({
index: migratedIndex,
index: SavedObjectsIndexPatterns,
});
const oldIndexResponse = await esClient.count({
index: '.kibana_1',
Expand Down
27 changes: 18 additions & 9 deletions src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import Hapi from '@hapi/hapi';
import h2o2 from '@hapi/h2o2';
import { URL } from 'url';
import type { SavedObject } from '@kbn/core-saved-objects-server';
import { SavedObject, SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server';
import type { InternalCoreSetup, InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
import { Root } from '@kbn/core-root-server-internal';
Expand All @@ -18,6 +18,7 @@ import {
createTestServers,
type TestElasticsearchUtils,
} from '@kbn/core-test-helpers-kbn-server';
import { kibanaPackageJson as pkg } from '@kbn/repo-info';
import {
declareGetRoute,
declareDeleteRoute,
Expand All @@ -30,6 +31,7 @@ import {
declarePostUpdateByQueryRoute,
declarePassthroughRoute,
setProxyInterrupt,
allCombinationsPermutations,
} from './repository_with_proxy_utils';

let esServer: TestElasticsearchUtils;
Expand Down Expand Up @@ -98,17 +100,24 @@ describe('404s from proxies', () => {
await hapiServer.register(h2o2);
// register specific routes to modify the response and a catch-all to relay the request/response as-is

declareGetRoute(hapiServer, esHostname, esPort);
declareDeleteRoute(hapiServer, esHostname, esPort);
declarePostUpdateRoute(hapiServer, esHostname, esPort);
allCombinationsPermutations(
SavedObjectsIndexPatterns.map((indexPattern) => `${indexPattern}_${pkg.version}`)
)
.map((indices) => indices.join(','))
.forEach((kbnIndexPath) => {
declareGetRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declareDeleteRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostUpdateRoute(hapiServer, esHostname, esPort, kbnIndexPath);

declareGetSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostPitRoute(hapiServer, esHostname, esPort, kbnIndexPath);
declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort, kbnIndexPath);
});

declareGetSearchRoute(hapiServer, esHostname, esPort);
declarePostSearchRoute(hapiServer, esHostname, esPort);
// register index-agnostic routes
declarePostBulkRoute(hapiServer, esHostname, esPort);
declarePostMgetRoute(hapiServer, esHostname, esPort);
declarePostPitRoute(hapiServer, esHostname, esPort);
declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort);

declarePassthroughRoute(hapiServer, esHostname, esPort);

await hapiServer.start();
Expand Down
67 changes: 57 additions & 10 deletions src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
*/
import Hapi from '@hapi/hapi';
import { IncomingMessage } from 'http';
import { kibanaPackageJson as pkg } from '@kbn/repo-info';

// proxy setup
const defaultProxyOptions = (hostname: string, port: string) => ({
Expand Down Expand Up @@ -52,10 +51,13 @@ const proxyOnResponseHandler = async (res: IncomingMessage, h: Hapi.ResponseTool
.code(404);
};

const kbnIndex = `.kibana_${pkg.version}`;

// GET /.kibana_8.0.0/_doc/{type*} route (repository.get calls)
export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declareGetRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'GET',
path: `/${kbnIndex}/_doc/{type*}`,
Expand All @@ -70,7 +72,12 @@ export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port:
},
});
// DELETE /.kibana_8.0.0/_doc/{type*} route (repository.delete calls)
export const declareDeleteRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declareDeleteRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'DELETE',
path: `/${kbnIndex}/_doc/{_id*}`,
Expand Down Expand Up @@ -133,7 +140,12 @@ export const declarePostMgetRoute = (hapiServer: Hapi.Server, hostname: string,
},
});
// GET _search route
export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declareGetSearchRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'GET',
path: `/${kbnIndex}/_search`,
Expand All @@ -149,7 +161,12 @@ export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string,
},
});
// POST _search route (`find` calls)
export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declarePostSearchRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
path: `/${kbnIndex}/_search`,
Expand All @@ -168,7 +185,12 @@ export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string
},
});
// POST _update
export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declarePostUpdateRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
path: `/${kbnIndex}/_update/{_id*}`,
Expand All @@ -187,7 +209,12 @@ export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string
},
});
// POST _pit
export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
export const declarePostPitRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
path: `/${kbnIndex}/_pit`,
Expand All @@ -209,7 +236,8 @@ export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, p
export const declarePostUpdateByQueryRoute = (
hapiServer: Hapi.Server,
hostname: string,
port: string
port: string,
kbnIndex: string
) =>
hapiServer.route({
method: 'POST',
Expand Down Expand Up @@ -244,3 +272,22 @@ export const declarePassthroughRoute = (hapiServer: Hapi.Server, hostname: strin
},
},
});

export function allCombinationsPermutations<T>(collection: T[]): T[][] {
const recur = (subcollection: T[], size: number): T[][] => {
if (size <= 0) {
return [[]];
}
const permutations: T[][] = [];
subcollection.forEach((value, index, array) => {
array = array.slice();
array.splice(index, 1);
recur(array, size - 1).forEach((permutation) => {
permutation.unshift(value);
permutations.push(permutation);
});
});
return permutations;
};
return collection.map((_, n) => recur(collection, n + 1)).flat();
}
103 changes: 103 additions & 0 deletions test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/mappings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,109 @@
"url": "c7f66a0df8b1b52f17c28c4adb111105",
"usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4",
"visualization": "f819cf6636b75c9e76ba733a0c6ef355"
},
"indexTypesMap": {
".kibana_task_manager": [
"task"
],
".kibana": [
"action_task_params",
"action",
"alert",
"api_key_pending_invalidation",
"apm-indices",
"apm-server-schema",
"apm-service-group",
"apm-telemetry",
"app_search_telemetry",
"application_usage_daily",
"application_usage_totals",
"canvas-element",
"canvas-workpad-template",
"canvas-workpad",
"cases-comments",
"cases-configure",
"cases-connector-mappings",
"cases-telemetry",
"cases-user-actions",
"cases",
"config-global",
"config",
"connector_token",
"core-usage-stats",
"csp-rule-template",
"dashboard",
"endpoint:user-artifact-manifest",
"endpoint:user-artifact",
"enterprise_search_telemetry",
"epm-packages-assets",
"epm-packages",
"event_loop_delays_daily",
"exception-list-agnostic",
"exception-list",
"file-upload-usage-collection-telemetry",
"file",
"fileShare",
"fleet-fleet-server-host",
"fleet-message-signing-keys",
"fleet-preconfiguration-deletion-record",
"fleet-proxy",
"graph-workspace",
"guided-onboarding-guide-state",
"guided-onboarding-plugin-state",
"index-pattern",
"infrastructure-monitoring-log-view",
"infrastructure-ui-source",
"ingest_manager_settings",
"ingest-agent-policies",
"ingest-download-sources",
"ingest-outputs",
"ingest-package-policies",
"inventory-view",
"kql-telemetry",
"legacy-url-alias",
"lens-ui-telemetry",
"lens",
"map",
"metrics-explorer-view",
"ml-job",
"ml-module",
"ml-trained-model",
"monitoring-telemetry",
"osquery-manager-usage-metric",
"osquery-pack-asset",
"osquery-pack",
"osquery-saved-query",
"query",
"rules-settings",
"sample-data-telemetry",
"search-session",
"search-telemetry",
"search",
"security-rule",
"security-solution-signals-migration",
"siem-detection-engine-rule-actions",
"siem-ui-timeline-note",
"siem-ui-timeline-pinned-event",
"siem-ui-timeline",
"slo",
"space",
"spaces-usage-stats",
"synthetics-monitor",
"synthetics-param",
"synthetics-privates-locations",
"tag",
"telemetry",
"ui-metric",
"upgrade-assistant-ml-upgrade-operation",
"upgrade-assistant-reindex-operation",
"uptime-dynamic-settings",
"uptime-synthetics-api-key",
"url",
"usage-counters",
"visualization",
"workplace_search_telemetry"
]
}
},
"dynamic": "strict",
Expand Down
Loading