From d6a9beb8442434b79fe3f9d484abfa158c5cc26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 14 Jun 2023 12:23:26 +0200 Subject: [PATCH 1/4] [APM] Add index.fast_refresh to `.apm-custom-link` --- .../scripts/shared/create_or_update_index.ts | 66 ------------------- .../create_agent_config_index.ts | 1 + .../custom_link/create_custom_link_index.ts | 5 ++ .../annotations/create_annotations_client.ts | 3 +- .../server/utils/create_or_update_index.ts | 49 ++++++++++---- 5 files changed, 43 insertions(+), 81 deletions(-) delete mode 100644 x-pack/plugins/apm/scripts/shared/create_or_update_index.ts diff --git a/x-pack/plugins/apm/scripts/shared/create_or_update_index.ts b/x-pack/plugins/apm/scripts/shared/create_or_update_index.ts deleted file mode 100644 index 39f398354422f..0000000000000 --- a/x-pack/plugins/apm/scripts/shared/create_or_update_index.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ESClient } from './get_es_client'; - -export async function createOrUpdateIndex({ - client, - clear, - indexName, - template, -}: { - client: ESClient; - clear: boolean; - indexName: string; - template: any; -}) { - if (clear) { - try { - await client.indices.delete({ - index: indexName, - }); - } catch (err) { - // 404 = index not found, totally okay - if (err.body.status !== 404) { - throw err; - } - } - } - - // Some settings are non-updateable and need to be removed. - const settings = { ...template.settings }; - delete settings?.index?.number_of_shards; - delete settings?.index?.sort; - - const indexExists = await client.indices.exists({ - index: indexName, - }); - - if (!indexExists) { - await client.indices.create({ - index: indexName, - body: template, - }); - } else { - await client.indices.close({ index: indexName }); - await Promise.all([ - template.mappings - ? client.indices.putMapping({ - index: indexName, - body: template.mappings, - }) - : Promise.resolve(undefined as any), - settings - ? client.indices.putSettings({ - index: indexName, - body: settings, - }) - : Promise.resolve(undefined as any), - ]); - await client.indices.open({ index: indexName }); - } -} diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts index 4aebdfaf03e6a..dcbfd604703c0 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts @@ -24,6 +24,7 @@ export async function createApmAgentConfigurationIndex({ client, logger, mappings, + settings: {}, }); } diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts b/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts index b8bfc3b3f2f72..7c5947db7ccac 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts @@ -25,6 +25,11 @@ export const createApmCustomLinkIndex = async ({ client, logger, mappings, + settings: { + index: { + fast_refresh: true, + }, + }, }); }; diff --git a/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts index fc74ffa880fc2..ce50d6b7a254a 100644 --- a/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts +++ b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts @@ -34,9 +34,10 @@ export function createAnnotationsClient(params: { const initIndex = () => createOrUpdateIndex({ index, - mappings, client: esClient, logger, + mappings, + settings: {}, }); function ensureGoldLicense any>(fn: T): T { diff --git a/x-pack/plugins/observability/server/utils/create_or_update_index.ts b/x-pack/plugins/observability/server/utils/create_or_update_index.ts index eaea86c18b19f..92258d5056b48 100644 --- a/x-pack/plugins/observability/server/utils/create_or_update_index.ts +++ b/x-pack/plugins/observability/server/utils/create_or_update_index.ts @@ -7,18 +7,23 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import pRetry from 'p-retry'; import { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { merge } from 'lodash'; export type Mappings = Required['body']['mappings'] & Required['body']; +type IndexSettings = Required['body']['settings']; + export async function createOrUpdateIndex({ index, mappings, + settings, client, logger, }: { index: string; mappings: Mappings; + settings: IndexSettings; client: ElasticsearchClient; logger: Logger; }) { @@ -34,23 +39,27 @@ export async function createOrUpdateIndex({ await pRetry( async () => { const indexExists = await client.indices.exists({ index }); - const result = indexExists + const results = indexExists ? await updateExistingIndex({ index, client, mappings, + settings, }) : await createNewIndex({ index, client, mappings, + settings, }); - if (!result.acknowledged) { - const bodyWithError: { body?: { error: any } } = result as any; - const resultError = JSON.stringify(bodyWithError?.body?.error); - throw new Error(resultError); - } + results.forEach((result) => { + if (!result.acknowledged) { + const bodyWithError: { body?: { error: any } } = result as any; + const resultError = JSON.stringify(bodyWithError?.body?.error); + throw new Error(resultError); + } + }); }, { onFailedAttempt: (e) => { @@ -64,36 +73,48 @@ export async function createOrUpdateIndex({ } } -function createNewIndex({ +async function createNewIndex({ index, client, mappings, + settings, }: { index: string; client: ElasticsearchClient; mappings: Required['body']['mappings']; + settings: Required['body']['settings']; }) { - return client.indices.create({ + const res = await client.indices.create({ index, body: { // auto_expand_replicas: Allows cluster to not have replicas for this index - settings: { index: { auto_expand_replicas: '0-1' } }, + settings: merge({ index: { auto_expand_replicas: '0-1' } }, settings), mappings, }, }); + + return [res]; } -function updateExistingIndex({ +async function updateExistingIndex({ index, client, mappings, + settings, }: { index: string; client: ElasticsearchClient; mappings: estypes.IndicesPutMappingRequest['body']; + settings: estypes.IndicesPutSettingsRequest['body']; }) { - return client.indices.putMapping({ - index, - body: mappings, - }); + return Promise.all([ + client.indices.putSettings({ + index, + body: settings, + }), + client.indices.putMapping({ + index, + body: mappings, + }), + ]); } From b324347eafa338549fe36c82f2b0b555330ceec5 Mon Sep 17 00:00:00 2001 From: "miriam.aparicio" Date: Tue, 11 Jul 2023 09:48:33 +0200 Subject: [PATCH 2/4] add fast_refresh setting just when creating custom links index --- .../create_agent_config_index.ts | 1 - .../annotations/create_annotations_client.ts | 1 - .../server/utils/create_or_update_index.ts | 20 ++++++------------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts index dcbfd604703c0..4aebdfaf03e6a 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts @@ -24,7 +24,6 @@ export async function createApmAgentConfigurationIndex({ client, logger, mappings, - settings: {}, }); } diff --git a/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts index ce50d6b7a254a..9c960ef09755d 100644 --- a/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts +++ b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts @@ -37,7 +37,6 @@ export function createAnnotationsClient(params: { client: esClient, logger, mappings, - settings: {}, }); function ensureGoldLicense any>(fn: T): T { diff --git a/x-pack/plugins/observability/server/utils/create_or_update_index.ts b/x-pack/plugins/observability/server/utils/create_or_update_index.ts index 92258d5056b48..5c8af99bcfbc2 100644 --- a/x-pack/plugins/observability/server/utils/create_or_update_index.ts +++ b/x-pack/plugins/observability/server/utils/create_or_update_index.ts @@ -23,7 +23,7 @@ export async function createOrUpdateIndex({ }: { index: string; mappings: Mappings; - settings: IndexSettings; + settings?: IndexSettings; client: ElasticsearchClient; logger: Logger; }) { @@ -44,7 +44,6 @@ export async function createOrUpdateIndex({ index, client, mappings, - settings, }) : await createNewIndex({ index, @@ -100,21 +99,14 @@ async function updateExistingIndex({ index, client, mappings, - settings, }: { index: string; client: ElasticsearchClient; mappings: estypes.IndicesPutMappingRequest['body']; - settings: estypes.IndicesPutSettingsRequest['body']; }) { - return Promise.all([ - client.indices.putSettings({ - index, - body: settings, - }), - client.indices.putMapping({ - index, - body: mappings, - }), - ]); + const res = await client.indices.putMapping({ + index, + body: mappings, + }); + return [res]; } From 539ec05cb3ece6633b705d84de695d6f776b478f Mon Sep 17 00:00:00 2001 From: "miriam.aparicio" Date: Tue, 11 Jul 2023 15:17:37 +0200 Subject: [PATCH 3/4] add feature flag for fast_refresh setting --- config/serverless.oblt.yml | 1 + .../plugins/apm/common/apm_feature_flags.ts | 5 +++++ .../apm_plugin/mock_apm_plugin_context.tsx | 1 + x-pack/plugins/apm/public/index.ts | 1 + x-pack/plugins/apm/server/index.ts | 10 +++++++++ x-pack/plugins/apm/server/plugin.ts | 3 ++- .../custom_link/create_custom_link_index.ts | 15 ++++++++----- .../server/utils/create_or_update_index.ts | 21 +++++++------------ 8 files changed, 38 insertions(+), 19 deletions(-) diff --git a/config/serverless.oblt.yml b/config/serverless.oblt.yml index 9f34fb10898a6..2928e75095897 100644 --- a/config/serverless.oblt.yml +++ b/config/serverless.oblt.yml @@ -35,6 +35,7 @@ xpack.apm.featureFlags.infraUiAvailable: false xpack.apm.featureFlags.migrationToFleetAvailable: false xpack.apm.featureFlags.sourcemapApiAvailable: false xpack.apm.featureFlags.storageExplorerAvailable: false +xpack.apm.featureFlags.fastRefreshAvailable: true # Specify in telemetry the project type telemetry.labels.serverless: observability diff --git a/x-pack/plugins/apm/common/apm_feature_flags.ts b/x-pack/plugins/apm/common/apm_feature_flags.ts index 0685555d02c1e..75dd58e7e66b8 100644 --- a/x-pack/plugins/apm/common/apm_feature_flags.ts +++ b/x-pack/plugins/apm/common/apm_feature_flags.ts @@ -16,6 +16,7 @@ export enum ApmFeatureFlagName { MigrationToFleetAvailable = 'migrationToFleetAvailable', SourcemapApiAvailable = 'sourcemapApiAvailable', StorageExplorerAvailable = 'storageExplorerAvailable', + FastRefreshAvailable = 'fastRefreshAvailable', } const apmFeatureFlagMap = { @@ -47,6 +48,10 @@ const apmFeatureFlagMap = { default: true, type: t.boolean, }, + [ApmFeatureFlagName.FastRefreshAvailable]: { + default: false, + type: t.boolean, + }, }; type ApmFeatureFlagMap = typeof apmFeatureFlagMap; diff --git a/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx b/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx index 0719f2dbd720d..10086aa2afd9d 100644 --- a/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx +++ b/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx @@ -79,6 +79,7 @@ const mockConfig: ConfigSchema = { migrationToFleetAvailable: true, sourcemapApiAvailable: true, storageExplorerAvailable: true, + fastRefreshAvailable: false, }, serverless: { enabled: false }, }; diff --git a/x-pack/plugins/apm/public/index.ts b/x-pack/plugins/apm/public/index.ts index 446bb61f181b6..a25c76e8e9e4a 100644 --- a/x-pack/plugins/apm/public/index.ts +++ b/x-pack/plugins/apm/public/index.ts @@ -24,6 +24,7 @@ export interface ConfigSchema { migrationToFleetAvailable: boolean; sourcemapApiAvailable: boolean; storageExplorerAvailable: boolean; + fastRefreshAvailable: boolean; }; serverless: { enabled: boolean; diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts index f79c83683a36b..9662fa7d9b0e6 100644 --- a/x-pack/plugins/apm/server/index.ts +++ b/x-pack/plugins/apm/server/index.ts @@ -23,6 +23,15 @@ const disabledOnServerless = schema.conditional( schema.oneOf([schema.literal(true)], { defaultValue: true }) ); +const enabledOnServerless = schema.conditional( + schema.contextRef('serverless'), + true, + schema.boolean({ + defaultValue: true, + }), + schema.oneOf([schema.literal(false)], { defaultValue: false }) +); + // All options should be documented in the APM configuration settings: https://github.com/elastic/kibana/blob/main/docs/settings/apm-settings.asciidoc // and be included on cloud allow list unless there are specific reasons not to const configSchema = schema.object({ @@ -88,6 +97,7 @@ const configSchema = schema.object({ migrationToFleetAvailable: disabledOnServerless, sourcemapApiAvailable: disabledOnServerless, storageExplorerAvailable: disabledOnServerless, + fastRefreshAvailable: enabledOnServerless, }), serverless: schema.object({ enabled: schema.conditional( diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 618c1b388bfab..d3979628f14b5 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -277,6 +277,7 @@ export class APMPlugin const logger = this.logger; const client = core.elasticsearch.client.asInternalUser; + const { featureFlags } = this.currentConfig; // create .apm-agent-configuration index without blocking start lifecycle createApmAgentConfigurationIndex({ client, logger }).catch((e) => { @@ -285,7 +286,7 @@ export class APMPlugin }); // create .apm-custom-link index without blocking start lifecycle - createApmCustomLinkIndex({ client, logger }).catch((e) => { + createApmCustomLinkIndex({ client, logger, featureFlags }).catch((e) => { logger.error('Failed to create .apm-custom-link index'); logger.error(e); }); diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts b/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts index 7c5947db7ccac..1586abb500be2 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts @@ -12,24 +12,29 @@ import { Mappings, } from '@kbn/observability-plugin/server'; import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices'; +import { ApmFeatureFlags } from '../../../../common/apm_feature_flags'; export const createApmCustomLinkIndex = async ({ client, logger, + featureFlags, }: { client: ElasticsearchClient; logger: Logger; + featureFlags: ApmFeatureFlags; }) => { return createOrUpdateIndex({ index: APM_CUSTOM_LINK_INDEX, client, logger, mappings, - settings: { - index: { - fast_refresh: true, - }, - }, + settings: featureFlags.fastRefreshAvailable + ? { + index: { + fast_refresh: true, + }, + } + : {}, }); }; diff --git a/x-pack/plugins/observability/server/utils/create_or_update_index.ts b/x-pack/plugins/observability/server/utils/create_or_update_index.ts index 5c8af99bcfbc2..1a158594a0ed2 100644 --- a/x-pack/plugins/observability/server/utils/create_or_update_index.ts +++ b/x-pack/plugins/observability/server/utils/create_or_update_index.ts @@ -39,7 +39,7 @@ export async function createOrUpdateIndex({ await pRetry( async () => { const indexExists = await client.indices.exists({ index }); - const results = indexExists + const result = indexExists ? await updateExistingIndex({ index, client, @@ -52,13 +52,11 @@ export async function createOrUpdateIndex({ settings, }); - results.forEach((result) => { - if (!result.acknowledged) { - const bodyWithError: { body?: { error: any } } = result as any; - const resultError = JSON.stringify(bodyWithError?.body?.error); - throw new Error(resultError); - } - }); + if (!result.acknowledged) { + const bodyWithError: { body?: { error: any } } = result as any; + const resultError = JSON.stringify(bodyWithError?.body?.error); + throw new Error(resultError); + } }, { onFailedAttempt: (e) => { @@ -83,7 +81,7 @@ async function createNewIndex({ mappings: Required['body']['mappings']; settings: Required['body']['settings']; }) { - const res = await client.indices.create({ + return await client.indices.create({ index, body: { // auto_expand_replicas: Allows cluster to not have replicas for this index @@ -91,8 +89,6 @@ async function createNewIndex({ mappings, }, }); - - return [res]; } async function updateExistingIndex({ @@ -104,9 +100,8 @@ async function updateExistingIndex({ client: ElasticsearchClient; mappings: estypes.IndicesPutMappingRequest['body']; }) { - const res = await client.indices.putMapping({ + return await client.indices.putMapping({ index, body: mappings, }); - return [res]; } From 27db0a1948fb183ce8efe8a3e2e4060425a7e679 Mon Sep 17 00:00:00 2001 From: "miriam.aparicio" Date: Tue, 11 Jul 2023 17:06:54 +0200 Subject: [PATCH 4/4] add feature flag to ftr config tests --- test/plugin_functional/test_suites/core_plugins/rendering.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts index e050ecf0a3ba8..879c2a0a5baa2 100644 --- a/test/plugin_functional/test_suites/core_plugins/rendering.ts +++ b/test/plugin_functional/test_suites/core_plugins/rendering.ts @@ -184,6 +184,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { 'xpack.apm.latestAgentVersionsUrl (string)', 'xpack.apm.featureFlags.agentConfigurationAvailable (any)', 'xpack.apm.featureFlags.configurableIndicesAvailable (any)', + 'xpack.apm.featureFlags.fastRefreshAvailable (any)', 'xpack.apm.featureFlags.infrastructureTabAvailable (any)', 'xpack.apm.featureFlags.infraUiAvailable (any)', 'xpack.apm.featureFlags.migrationToFleetAvailable (any)',