diff --git a/x-pack/plugins/monitoring/common/ccs_utils.ts b/x-pack/plugins/monitoring/common/ccs_utils.ts index 7efe6e43ddbbd8..982189a1e4a977 100644 --- a/x-pack/plugins/monitoring/common/ccs_utils.ts +++ b/x-pack/plugins/monitoring/common/ccs_utils.ts @@ -13,32 +13,17 @@ type Config = Partial & { get?: (key: string) => any; }; -export function appendMetricbeatIndex( - config: Config, - indexPattern: string, - ccs?: string, - bypass: boolean = false -) { - if (bypass) { - return indexPattern; - } - // Leverage this function to also append the dynamic metricbeat index too - let mbIndex = null; +export function getConfigCcs(config: Config): boolean { + let ccsEnabled = false; // TODO: NP // This function is called with both NP config and LP config if (isFunction(config.get)) { - mbIndex = config.get('monitoring.ui.metricbeat.index'); + ccsEnabled = config.get('monitoring.ui.ccs.enabled'); } else { - mbIndex = get(config, 'ui.metricbeat.index'); - } - - if (ccs) { - mbIndex = `${mbIndex},${ccs}:${mbIndex}`; + ccsEnabled = get(config, 'ui.ccs.enabled'); } - - return `${indexPattern},${mbIndex}`; + return ccsEnabled; } - /** * Prefix all comma separated index patterns within the original {@code indexPattern}. * @@ -50,28 +35,10 @@ export function appendMetricbeatIndex( * @param {String} ccs The optional cluster-prefix to prepend. * @return {String} The index pattern with the {@code cluster} prefix appropriately prepended. */ -export function prefixIndexPattern( - config: Config, - indexPattern: string, - ccs?: string, - monitoringIndicesOnly: boolean = false -) { - let ccsEnabled = false; - // TODO: NP - // This function is called with both NP config and LP config - if (isFunction(config.get)) { - ccsEnabled = config.get('monitoring.ui.ccs.enabled'); - } else { - ccsEnabled = get(config, 'ui.ccs.enabled'); - } - +export function prefixIndexPattern(config: Config, indexPattern: string, ccs?: string) { + const ccsEnabled = getConfigCcs(config); if (!ccsEnabled || !ccs) { - return appendMetricbeatIndex( - config, - indexPattern, - ccsEnabled ? ccs : undefined, - monitoringIndicesOnly - ); + return indexPattern; } const patterns = indexPattern.split(','); @@ -79,15 +46,9 @@ export function prefixIndexPattern( // if a wildcard is used, then we also want to search the local indices if (ccs === '*') { - return appendMetricbeatIndex( - config, - `${prefixedPattern},${indexPattern}`, - ccs, - monitoringIndicesOnly - ); + return `${prefixedPattern},${indexPattern}`; } - - return appendMetricbeatIndex(config, prefixedPattern, ccs, monitoringIndicesOnly); + return prefixedPattern; } /** diff --git a/x-pack/plugins/monitoring/common/constants.ts b/x-pack/plugins/monitoring/common/constants.ts index 67cc86c013ca2d..96f66fc3d41778 100644 --- a/x-pack/plugins/monitoring/common/constants.ts +++ b/x-pack/plugins/monitoring/common/constants.ts @@ -132,6 +132,9 @@ export const INDEX_PATTERN_ELASTICSEARCH = '.monitoring-es-*'; // ECS-compliant patterns (metricbeat >8 and agent) export const INDEX_PATTERN_ELASTICSEARCH_ECS = '.monitoring-es-8-*'; export const INDEX_PATTERN_ENTERPRISE_SEARCH = '.monitoring-ent-search-*'; +export const DS_INDEX_PATTERN_METRICS = 'metrics'; +export const DS_INDEX_PATTERN_LOGS = 'logs'; +export const DS_INDEX_PATTERN_ES = 'elasticsearch'; // This is the unique token that exists in monitoring indices collected by metricbeat export const METRICBEAT_INDEX_NAME_UNIQUE_TOKEN = '-mb-'; @@ -586,3 +589,12 @@ export const ALERT_EMAIL_SERVICES = ['gmail', 'hotmail', 'icloud', 'outlook365', export const SAVED_OBJECT_TELEMETRY = 'monitoring-telemetry'; export const TELEMETRY_METRIC_BUTTON_CLICK = 'btnclick__'; + +export type INDEX_PATTERN_TYPES = + | 'elasticsearch' + | 'kibana' + | 'logstash' + | 'beats' + | 'enterprisesearch'; + +export type DS_INDEX_PATTERN_TYPES = typeof DS_INDEX_PATTERN_METRICS | typeof DS_INDEX_PATTERN_LOGS; diff --git a/x-pack/plugins/monitoring/server/alerts/base_rule.ts b/x-pack/plugins/monitoring/server/alerts/base_rule.ts index f05077ec4bb006..d13e6d9ed7f9b2 100644 --- a/x-pack/plugins/monitoring/server/alerts/base_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/base_rule.ts @@ -28,10 +28,7 @@ import { CommonAlertParams, } from '../../common/types/alerts'; import { fetchClusters } from '../lib/alerts/fetch_clusters'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../common/constants'; import { AlertSeverity } from '../../common/enums'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { parseDuration } from '../../../alerting/common'; import { Globals } from '../static_globals'; @@ -226,23 +223,14 @@ export class BaseRule { ); const esClient = services.scopedClusterClient.asCurrentUser; - const availableCcs = Globals.app.config.ui.ccs.enabled; - const clusters = await this.fetchClusters(esClient, params as CommonAlertParams, availableCcs); - const data = await this.fetchData(params, esClient, clusters, availableCcs); + const clusters = await this.fetchClusters(esClient, params as CommonAlertParams); + const data = await this.fetchData(params, esClient, clusters); return await this.processData(data, clusters, services, state); } - protected async fetchClusters( - esClient: ElasticsearchClient, - params: CommonAlertParams, - ccs?: boolean - ) { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (ccs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, ccs); - } + protected async fetchClusters(esClient: ElasticsearchClient, params: CommonAlertParams) { if (!params.limit) { - return await fetchClusters(esClient, esIndexPattern); + return await fetchClusters(esClient); } const limit = parseDuration(params.limit); const rangeFilter = this.ruleOptions.fetchClustersRange @@ -253,14 +241,13 @@ export class BaseRule { }, } : undefined; - return await fetchClusters(esClient, esIndexPattern, rangeFilter); + return await fetchClusters(esClient, rangeFilter); } protected async fetchData( params: CommonAlertParams | unknown, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise> { throw new Error('Child classes must implement `fetchData`'); } diff --git a/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.test.ts index 8dd4623bfd7e43..ed4ba69b8e2547 100644 --- a/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.test.ts @@ -39,7 +39,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.ts b/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.ts index 9b2f9b9fb3ed75..705d0c6b9c87f1 100644 --- a/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/ccr_read_exceptions_rule.ts @@ -22,18 +22,12 @@ import { CCRReadExceptionsStats, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - INDEX_PATTERN_ELASTICSEARCH, - RULE_CCR_READ_EXCEPTIONS, - RULE_DETAILS, -} from '../../common/constants'; +import { RULE_CCR_READ_EXCEPTIONS, RULE_DETAILS } from '../../common/constants'; import { fetchCCRReadExceptions } from '../lib/alerts/fetch_ccr_read_exceptions'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { parseDuration } from '../../../alerting/common/parse_duration'; import { SanitizedAlert, RawAlertInstance } from '../../../alerting/common'; import { AlertingDefaults, createLink } from './alert_helpers'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { Globals } from '../static_globals'; export class CCRReadExceptionsRule extends BaseRule { @@ -72,20 +66,14 @@ export class CCRReadExceptionsRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } const { duration: durationString } = params; const duration = parseDuration(durationString); const endMs = +new Date(); const startMs = endMs - duration; const stats = await fetchCCRReadExceptions( esClient, - esIndexPattern, startMs, endMs, Globals.app.config.ui.max_bucket_size, diff --git a/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.test.ts index 5d209f7fc4a81f..85030657825c4f 100644 --- a/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.test.ts @@ -21,7 +21,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.ts b/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.ts index e85fb33cd76bda..b8810196c833a1 100644 --- a/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/cluster_health_rule.ts @@ -19,17 +19,10 @@ import { AlertInstanceState, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - RULE_CLUSTER_HEALTH, - LEGACY_RULE_DETAILS, - INDEX_PATTERN_ELASTICSEARCH, -} from '../../common/constants'; +import { RULE_CLUSTER_HEALTH, LEGACY_RULE_DETAILS } from '../../common/constants'; import { AlertMessageTokenType, AlertClusterHealthType, AlertSeverity } from '../../common/enums'; import { AlertingDefaults } from './alert_helpers'; import { SanitizedAlert } from '../../../alerting/common'; -import { Globals } from '../static_globals'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { fetchClusterHealth } from '../lib/alerts/fetch_cluster_health'; const RED_STATUS_MESSAGE = i18n.translate('xpack.monitoring.alerts.clusterHealth.redMessage', { @@ -66,19 +59,9 @@ export class ClusterHealthRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } - const healths = await fetchClusterHealth( - esClient, - clusters, - esIndexPattern, - params.filterQuery - ); + const healths = await fetchClusterHealth(esClient, clusters, params.filterQuery); return healths.map((clusterHealth) => { const shouldFire = clusterHealth.health !== AlertClusterHealthType.Green; const severity = diff --git a/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.test.ts index 9b19c1ddeb7d11..bcd2c0cbb58102 100644 --- a/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.test.ts @@ -27,7 +27,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.ts b/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.ts index b41783d449c02b..fa4b64fd997c34 100644 --- a/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/cpu_usage_rule.ts @@ -23,16 +23,14 @@ import { CommonAlertFilter, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { INDEX_PATTERN_ELASTICSEARCH, RULE_CPU_USAGE, RULE_DETAILS } from '../../common/constants'; +import { RULE_CPU_USAGE, RULE_DETAILS } from '../../common/constants'; // @ts-ignore import { ROUNDED_FLOAT } from '../../common/formatting'; import { fetchCpuUsageNodeStats } from '../lib/alerts/fetch_cpu_usage_node_stats'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { RawAlertInstance, SanitizedAlert } from '../../../alerting/common'; import { parseDuration } from '../../../alerting/common/parse_duration'; import { AlertingDefaults, createLink } from './alert_helpers'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { Globals } from '../static_globals'; export class CpuUsageRule extends BaseRule { @@ -60,20 +58,14 @@ export class CpuUsageRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } const duration = parseDuration(params.duration); const endMs = +new Date(); const startMs = endMs - duration; const stats = await fetchCpuUsageNodeStats( esClient, clusters, - esIndexPattern, startMs, endMs, Globals.app.config.ui.max_bucket_size, diff --git a/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.test.ts index 63ff6a7ccab93a..daaded1c18c803 100644 --- a/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.test.ts @@ -40,7 +40,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.ts b/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.ts index 17dff8ea6a9dd4..1e06f0649d1078 100644 --- a/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/disk_usage_rule.ts @@ -23,15 +23,13 @@ import { CommonAlertFilter, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { INDEX_PATTERN_ELASTICSEARCH, RULE_DISK_USAGE, RULE_DETAILS } from '../../common/constants'; +import { RULE_DISK_USAGE, RULE_DETAILS } from '../../common/constants'; // @ts-ignore import { ROUNDED_FLOAT } from '../../common/formatting'; import { fetchDiskUsageNodeStats } from '../lib/alerts/fetch_disk_usage_node_stats'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { RawAlertInstance, SanitizedAlert } from '../../../alerting/common'; import { AlertingDefaults, createLink } from './alert_helpers'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { Globals } from '../static_globals'; export class DiskUsageRule extends BaseRule { @@ -59,18 +57,12 @@ export class DiskUsageRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } const { duration, threshold } = params; const stats = await fetchDiskUsageNodeStats( esClient, clusters, - esIndexPattern, duration as string, Globals.app.config.ui.max_bucket_size, params.filterQuery diff --git a/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.test.ts index 12fa54f34e3c4d..4531c5f0f1ffc1 100644 --- a/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.test.ts @@ -28,7 +28,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.ts b/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.ts index b873a20c874b53..9d89f827f9b10d 100644 --- a/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_rule.ts @@ -18,17 +18,11 @@ import { AlertVersions, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - RULE_ELASTICSEARCH_VERSION_MISMATCH, - LEGACY_RULE_DETAILS, - INDEX_PATTERN_ELASTICSEARCH, -} from '../../common/constants'; +import { RULE_ELASTICSEARCH_VERSION_MISMATCH, LEGACY_RULE_DETAILS } from '../../common/constants'; import { AlertSeverity } from '../../common/enums'; import { AlertingDefaults } from './alert_helpers'; import { SanitizedAlert } from '../../../alerting/common'; import { Globals } from '../static_globals'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { fetchElasticsearchVersions } from '../lib/alerts/fetch_elasticsearch_versions'; export class ElasticsearchVersionMismatchRule extends BaseRule { @@ -55,17 +49,11 @@ export class ElasticsearchVersionMismatchRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } const elasticsearchVersions = await fetchElasticsearchVersions( esClient, clusters, - esIndexPattern, Globals.app.config.ui.max_bucket_size, params.filterQuery ); diff --git a/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.test.ts index 01016a7c02ae20..b4444c9088073c 100644 --- a/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.test.ts @@ -28,7 +28,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.ts b/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.ts index 79f449f8e7ef72..24182c4a545d3d 100644 --- a/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_rule.ts @@ -18,17 +18,11 @@ import { AlertVersions, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - RULE_KIBANA_VERSION_MISMATCH, - LEGACY_RULE_DETAILS, - INDEX_PATTERN_KIBANA, -} from '../../common/constants'; +import { RULE_KIBANA_VERSION_MISMATCH, LEGACY_RULE_DETAILS } from '../../common/constants'; import { AlertSeverity } from '../../common/enums'; import { AlertingDefaults } from './alert_helpers'; import { SanitizedAlert } from '../../../alerting/common'; import { Globals } from '../static_globals'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { fetchKibanaVersions } from '../lib/alerts/fetch_kibana_versions'; export class KibanaVersionMismatchRule extends BaseRule { @@ -68,17 +62,11 @@ export class KibanaVersionMismatchRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let kibanaIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_KIBANA); - if (availableCcs) { - kibanaIndexPattern = getCcsIndexPattern(kibanaIndexPattern, availableCcs); - } const kibanaVersions = await fetchKibanaVersions( esClient, clusters, - kibanaIndexPattern, Globals.app.config.ui.max_bucket_size, params.filterQuery ); diff --git a/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts index f7d6081edd306c..0460064b4f7c55 100644 --- a/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.test.ts @@ -40,7 +40,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.ts b/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.ts index 3009995e2f292f..92be43b9c06c0f 100644 --- a/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/large_shard_size_rule.ts @@ -22,17 +22,11 @@ import { IndexShardSizeStats, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - INDEX_PATTERN_ELASTICSEARCH, - RULE_LARGE_SHARD_SIZE, - RULE_DETAILS, -} from '../../common/constants'; +import { RULE_LARGE_SHARD_SIZE, RULE_DETAILS } from '../../common/constants'; import { fetchIndexShardSize } from '../lib/alerts/fetch_index_shard_size'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { SanitizedAlert, RawAlertInstance } from '../../../alerting/common'; import { AlertingDefaults, createLink } from './alert_helpers'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { Globals } from '../static_globals'; export class LargeShardSizeRule extends BaseRule { @@ -60,19 +54,13 @@ export class LargeShardSizeRule extends BaseRule { protected async fetchData( params: CommonAlertParams & { indexPattern: string }, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } const { threshold, indexPattern: shardIndexPatterns } = params; const stats = await fetchIndexShardSize( esClient, clusters, - esIndexPattern, threshold!, shardIndexPatterns, Globals.app.config.ui.max_bucket_size, diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.test.ts index b29a5fc4661d74..86a6f666fcf87c 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.test.ts @@ -34,7 +34,6 @@ jest.mock('../static_globals', () => ({ ui: { show_license_expiration: true, ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.ts index fc050bd6780129..3a837a125a523b 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration_rule.ts @@ -21,17 +21,11 @@ import { AlertLicenseState, } from '../../common/types/alerts'; import { AlertExecutorOptions, AlertInstance } from '../../../alerting/server'; -import { - RULE_LICENSE_EXPIRATION, - LEGACY_RULE_DETAILS, - INDEX_PATTERN_ELASTICSEARCH, -} from '../../common/constants'; +import { RULE_LICENSE_EXPIRATION, LEGACY_RULE_DETAILS } from '../../common/constants'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { AlertingDefaults } from './alert_helpers'; import { SanitizedAlert } from '../../../alerting/common'; import { Globals } from '../static_globals'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { fetchLicenses } from '../lib/alerts/fetch_licenses'; const EXPIRES_DAYS = [60, 30, 14, 7]; @@ -80,14 +74,9 @@ export class LicenseExpirationRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } - const licenses = await fetchLicenses(esClient, clusters, esIndexPattern, params.filterQuery); + const licenses = await fetchLicenses(esClient, clusters, params.filterQuery); return licenses.map((license) => { const { clusterUuid, type, expiryDateMS, status, ccs } = license; diff --git a/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.test.ts index 20f64b65ba1f05..857a9bf5bfa791 100644 --- a/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.test.ts @@ -29,7 +29,6 @@ jest.mock('../static_globals', () => ({ ui: { show_license_expiration: true, ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.ts b/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.ts index 6d7c06c1c1e07c..ee3e5838d7d35c 100644 --- a/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_rule.ts @@ -18,17 +18,11 @@ import { AlertVersions, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - RULE_LOGSTASH_VERSION_MISMATCH, - LEGACY_RULE_DETAILS, - INDEX_PATTERN_LOGSTASH, -} from '../../common/constants'; +import { RULE_LOGSTASH_VERSION_MISMATCH, LEGACY_RULE_DETAILS } from '../../common/constants'; import { AlertSeverity } from '../../common/enums'; import { AlertingDefaults } from './alert_helpers'; import { SanitizedAlert } from '../../../alerting/common'; import { Globals } from '../static_globals'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { fetchLogstashVersions } from '../lib/alerts/fetch_logstash_versions'; export class LogstashVersionMismatchRule extends BaseRule { @@ -55,17 +49,11 @@ export class LogstashVersionMismatchRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let logstashIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_LOGSTASH); - if (availableCcs) { - logstashIndexPattern = getCcsIndexPattern(logstashIndexPattern, availableCcs); - } const logstashVersions = await fetchLogstashVersions( esClient, clusters, - logstashIndexPattern, Globals.app.config.ui.max_bucket_size, params.filterQuery ); diff --git a/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.test.ts index 8547f126a02d6e..6e7aff2ae8fb4e 100644 --- a/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.test.ts @@ -27,7 +27,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.ts b/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.ts index 25b12379f8d3a5..06ecf4bb450c87 100644 --- a/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/memory_usage_rule.ts @@ -23,19 +23,13 @@ import { CommonAlertFilter, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - INDEX_PATTERN_ELASTICSEARCH, - RULE_MEMORY_USAGE, - RULE_DETAILS, -} from '../../common/constants'; +import { RULE_MEMORY_USAGE, RULE_DETAILS } from '../../common/constants'; // @ts-ignore import { ROUNDED_FLOAT } from '../../common/formatting'; import { fetchMemoryUsageNodeStats } from '../lib/alerts/fetch_memory_usage_node_stats'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { RawAlertInstance, SanitizedAlert } from '../../../alerting/common'; import { AlertingDefaults, createLink } from './alert_helpers'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { parseDuration } from '../../../alerting/common/parse_duration'; import { Globals } from '../static_globals'; @@ -64,13 +58,8 @@ export class MemoryUsageRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } const { duration, threshold } = params; const parsedDuration = parseDuration(duration as string); const endMs = +new Date(); @@ -79,7 +68,6 @@ export class MemoryUsageRule extends BaseRule { const stats = await fetchMemoryUsageNodeStats( esClient, clusters, - esIndexPattern, startMs, endMs, Globals.app.config.ui.max_bucket_size, diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.test.ts index 66259b9fffac92..a8a96a61a4b251 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.test.ts @@ -29,7 +29,6 @@ jest.mock('../static_globals', () => ({ ui: { show_license_expiration: true, ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.ts index 50ba9fa9e35864..fa7cbe009712a0 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_rule.ts @@ -20,12 +20,10 @@ import { AlertNodeState, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { INDEX_PATTERN, RULE_MISSING_MONITORING_DATA, RULE_DETAILS } from '../../common/constants'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; +import { RULE_MISSING_MONITORING_DATA, RULE_DETAILS } from '../../common/constants'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { RawAlertInstance, SanitizedAlert } from '../../../alerting/common'; import { parseDuration } from '../../../alerting/common/parse_duration'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { fetchMissingMonitoringData } from '../lib/alerts/fetch_missing_monitoring_data'; import { AlertingDefaults, createLink } from './alert_helpers'; import { Globals } from '../static_globals'; @@ -59,20 +57,14 @@ export class MissingMonitoringDataRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let indexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN); - if (availableCcs) { - indexPattern = getCcsIndexPattern(indexPattern, availableCcs); - } const duration = parseDuration(params.duration); const limit = parseDuration(params.limit!); const now = +new Date(); const missingData = await fetchMissingMonitoringData( esClient, clusters, - indexPattern, Globals.app.config.ui.max_bucket_size, now, now - limit - LIMIT_BUFFER, diff --git a/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.test.ts index 5145a4e8476af3..3e24df3a6ef151 100644 --- a/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.test.ts @@ -34,7 +34,6 @@ jest.mock('../static_globals', () => ({ config: { ui: { ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.ts b/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.ts index 545d6331d225e2..82cf91e91b52a9 100644 --- a/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.ts +++ b/x-pack/plugins/monitoring/server/alerts/nodes_changed_rule.ts @@ -20,19 +20,11 @@ import { AlertNodesChangedState, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { - RULE_NODES_CHANGED, - LEGACY_RULE_DETAILS, - INDEX_PATTERN_ELASTICSEARCH, -} from '../../common/constants'; +import { RULE_NODES_CHANGED, LEGACY_RULE_DETAILS } from '../../common/constants'; import { AlertingDefaults } from './alert_helpers'; import { SanitizedAlert } from '../../../alerting/common'; -import { Globals } from '../static_globals'; import { fetchNodesFromClusterStats } from '../lib/alerts/fetch_nodes_from_cluster_stats'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { AlertSeverity } from '../../common/enums'; - interface AlertNodesChangedStates { removed: AlertClusterStatsNode[]; added: AlertClusterStatsNode[]; @@ -104,17 +96,11 @@ export class NodesChangedRule extends BaseRule { protected async fetchData( params: CommonAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } const nodesFromClusterStats = await fetchNodesFromClusterStats( esClient, clusters, - esIndexPattern, params.filterQuery ); return nodesFromClusterStats.map((nodes) => { diff --git a/x-pack/plugins/monitoring/server/alerts/thread_pool_rejections_rule_base.ts b/x-pack/plugins/monitoring/server/alerts/thread_pool_rejections_rule_base.ts index e6c2002eaff87f..0cca5eb81c95f2 100644 --- a/x-pack/plugins/monitoring/server/alerts/thread_pool_rejections_rule_base.ts +++ b/x-pack/plugins/monitoring/server/alerts/thread_pool_rejections_rule_base.ts @@ -21,13 +21,10 @@ import { AlertThreadPoolRejectionsStats, } from '../../common/types/alerts'; import { AlertInstance } from '../../../alerting/server'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../common/constants'; import { fetchThreadPoolRejectionStats } from '../lib/alerts/fetch_thread_pool_rejections_stats'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; import { Alert, RawAlertInstance } from '../../../alerting/common'; import { AlertingDefaults, createLink } from './alert_helpers'; -import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; import { Globals } from '../static_globals'; type ActionVariables = Array<{ name: string; description: string }>; @@ -70,20 +67,13 @@ export class ThreadPoolRejectionsRuleBase extends BaseRule { protected async fetchData( params: ThreadPoolRejectionsAlertParams, esClient: ElasticsearchClient, - clusters: AlertCluster[], - availableCcs: boolean + clusters: AlertCluster[] ): Promise { - let esIndexPattern = appendMetricbeatIndex(Globals.app.config, INDEX_PATTERN_ELASTICSEARCH); - if (availableCcs) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } - const { threshold, duration } = params; const stats = await fetchThreadPoolRejectionStats( esClient, clusters, - esIndexPattern, Globals.app.config.ui.max_bucket_size, this.threadPoolType, duration, diff --git a/x-pack/plugins/monitoring/server/alerts/thread_pool_search_rejections_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/thread_pool_search_rejections_rule.test.ts index 351980d3f385df..63a02088b9b659 100644 --- a/x-pack/plugins/monitoring/server/alerts/thread_pool_search_rejections_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/thread_pool_search_rejections_rule.test.ts @@ -29,7 +29,6 @@ jest.mock('../static_globals', () => ({ ui: { show_license_expiration: true, ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/alerts/thread_pool_write_rejections_rule.test.ts b/x-pack/plugins/monitoring/server/alerts/thread_pool_write_rejections_rule.test.ts index 79896d11da2c31..da4c7ffaeffa09 100644 --- a/x-pack/plugins/monitoring/server/alerts/thread_pool_write_rejections_rule.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/thread_pool_write_rejections_rule.test.ts @@ -29,7 +29,6 @@ jest.mock('../static_globals', () => ({ ui: { show_license_expiration: true, ccs: { enabled: true }, - metricbeat: { index: 'metricbeat-*' }, container: { elasticsearch: { enabled: false } }, }, }, diff --git a/x-pack/plugins/monitoring/server/config.test.ts b/x-pack/plugins/monitoring/server/config.test.ts index 036ade38607e97..22e7b74368ebfa 100644 --- a/x-pack/plugins/monitoring/server/config.test.ts +++ b/x-pack/plugins/monitoring/server/config.test.ts @@ -102,9 +102,6 @@ describe('config schema', () => { "index": "filebeat-*", }, "max_bucket_size": 10000, - "metricbeat": Object { - "index": "metricbeat-*", - }, "min_interval_seconds": 10, "show_license_expiration": true, }, diff --git a/x-pack/plugins/monitoring/server/config.ts b/x-pack/plugins/monitoring/server/config.ts index 835ad30de7cbe7..3facfd97319f2c 100644 --- a/x-pack/plugins/monitoring/server/config.ts +++ b/x-pack/plugins/monitoring/server/config.ts @@ -32,9 +32,6 @@ export const configSchema = schema.object({ logs: schema.object({ index: schema.string({ defaultValue: 'filebeat-*' }), }), - metricbeat: schema.object({ - index: schema.string({ defaultValue: 'metricbeat-*' }), - }), max_bucket_size: schema.number({ defaultValue: 10000 }), elasticsearch: monitoringElasticsearchConfigSchema, container: schema.object({ diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts index 4c454637bf8bb3..cbbfe64f5e3e22 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts @@ -11,8 +11,6 @@ import { MonitoringConfig } from '../../config'; import { getStackProductsUsage } from './lib/get_stack_products_usage'; import { fetchLicenseType } from './lib/fetch_license_type'; import { MonitoringUsage, StackProductUsage, MonitoringClusterStackProductUsage } from './types'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../../common/constants'; -import { getCcsIndexPattern } from '../../lib/alerts/get_ccs_index_pattern'; import { fetchClusters } from '../../lib/alerts/fetch_clusters'; export function getMonitoringUsageCollector( @@ -106,8 +104,7 @@ export function getMonitoringUsageCollector( : getClient().asInternalUser; const usageClusters: MonitoringClusterStackProductUsage[] = []; const availableCcs = config.ui.ccs.enabled; - const elasticsearchIndex = getCcsIndexPattern(INDEX_PATTERN_ELASTICSEARCH, availableCcs); - const clusters = await fetchClusters(callCluster, elasticsearchIndex); + const clusters = await fetchClusters(callCluster); for (const cluster of clusters) { const license = await fetchLicenseType(callCluster, availableCcs, cluster.clusterUuid); const stackProducts = await getStackProductsUsage( diff --git a/x-pack/plugins/monitoring/server/lib/alerts/append_mb_index.ts b/x-pack/plugins/monitoring/server/lib/alerts/append_mb_index.ts deleted file mode 100644 index c8713f70ea5cf9..00000000000000 --- a/x-pack/plugins/monitoring/server/lib/alerts/append_mb_index.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { MonitoringConfig } from '../../config'; - -export function appendMetricbeatIndex(config: MonitoringConfig, indexPattern: string) { - return `${indexPattern},${config.ui.metricbeat.index}`; -} diff --git a/x-pack/plugins/monitoring/server/lib/alerts/create_dataset_query_filter.ts b/x-pack/plugins/monitoring/server/lib/alerts/create_dataset_query_filter.ts new file mode 100644 index 00000000000000..7d23240bc1e943 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/create_dataset_query_filter.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const createDatasetFilter = (legacyType: string, dataset: string) => ({ + bool: { + should: [ + { + term: { + type: legacyType, + }, + }, + { + term: { + 'data_stream.dataset': dataset, + }, + }, + ], + minimum_should_match: 1, + }, +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts index e7a5923207d604..d9dd035b92c4d5 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts @@ -8,17 +8,26 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { CCRReadExceptionsStats } from '../../../common/types/alerts'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; export async function fetchCCRReadExceptions( esClient: ElasticsearchClient, - index: string, startMs: number, endMs: number, size: number, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'ccr', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations.remote_clusters.buckets'], body: { size: 0, @@ -35,11 +44,7 @@ export async function fetchCCRReadExceptions( }, }, }, - { - term: { - type: 'ccr_stats', - }, - }, + createDatasetFilter('ccr_stats', 'elasticsearch.ccr'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts index 2739e23245bdee..ea87608a14ef03 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts @@ -10,6 +10,18 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { elasticsearchClientMock } from '../../../../../../src/core/server/elasticsearch/client/mocks'; import { fetchClusterHealth } from './fetch_cluster_health'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchClusterHealth', () => { it('should return the cluster health', async () => { const status = 'green'; @@ -34,9 +46,8 @@ describe('fetchClusterHealth', () => { ); const clusters = [{ clusterUuid, clusterName: 'foo' }]; - const index = '.monitoring-es-*'; - const health = await fetchClusterHealth(esClient, clusters, index); + const health = await fetchClusterHealth(esClient, clusters); expect(health).toEqual([ { health: status, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts index b2004f0c7c7106..59fd379344b4c9 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts @@ -7,15 +7,24 @@ import { ElasticsearchClient } from 'kibana/server'; import { AlertCluster, AlertClusterHealth } from '../../../common/types/alerts'; import { ElasticsearchSource, ElasticsearchResponse } from '../../../common/types/es'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; export async function fetchClusterHealth( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'cluster_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: [ 'hits.hits._source.cluster_state.status', 'hits.hits._source.cluster_uuid', @@ -39,11 +48,7 @@ export async function fetchClusterHealth( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - { - term: { - type: 'cluster_stats', - }, - }, + createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts index c46ec424b2cd37..fa4fa0269fb1b2 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts @@ -11,6 +11,18 @@ import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mo import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { fetchClusters } from './fetch_clusters'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchClusters', () => { const clusterUuid = '1sdfds734'; const clusterName = 'monitoring'; @@ -31,8 +43,7 @@ describe('fetchClusters', () => { }, } as estypes.SearchResponse) ); - const index = '.monitoring-es-*'; - const result = await fetchClusters(esClient, index); + const result = await fetchClusters(esClient); expect(result).toEqual([{ clusterUuid, clusterName }]); }); @@ -60,15 +71,13 @@ describe('fetchClusters', () => { }, } as estypes.SearchResponse) ); - const index = '.monitoring-es-*'; - const result = await fetchClusters(esClient, index); + const result = await fetchClusters(esClient); expect(result).toEqual([{ clusterUuid, clusterName: metadataName }]); }); it('should limit the time period in the query', async () => { const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; - const index = '.monitoring-es-*'; - await fetchClusters(esClient, index); + await fetchClusters(esClient); const params = esClient.search.mock.calls[0][0] as any; expect(params?.body?.query.bool.filter[1].range.timestamp.gte).toBe('now-2m'); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts index 984cdee4e1b7b4..76bd81c24145f4 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts @@ -8,6 +8,10 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { AlertCluster } from '../../../common/types/alerts'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; interface RangeFilter { [field: string]: { @@ -18,11 +22,15 @@ interface RangeFilter { export async function fetchClusters( esClient: ElasticsearchClient, - index: string, rangeFilter: RangeFilter = { timestamp: { gte: 'now-2m' } } ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: [ 'hits.hits._source.cluster_settings.cluster.metadata.display_name', 'hits.hits._source.cluster_uuid', @@ -33,11 +41,7 @@ export async function fetchClusters( query: { bool: { filter: [ - { - term: { - type: 'cluster_stats', - }, - }, + createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), { range: rangeFilter, }, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts index a67a5e679cc334..fd7b61b5f92fff 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts @@ -10,6 +10,18 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { elasticsearchClientMock } from '../../../../../../src/core/server/elasticsearch/client/mocks'; import { fetchCpuUsageNodeStats } from './fetch_cpu_usage_node_stats'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchCpuUsageNodeStats', () => { const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; const clusters = [ @@ -18,7 +30,6 @@ describe('fetchCpuUsageNodeStats', () => { clusterName: 'test', }, ]; - const index = '.monitoring-es-*'; const startMs = 0; const endMs = 0; const size = 10; @@ -62,7 +73,7 @@ describe('fetchCpuUsageNodeStats', () => { }, }) ); - const result = await fetchCpuUsageNodeStats(esClient, clusters, index, startMs, endMs, size); + const result = await fetchCpuUsageNodeStats(esClient, clusters, startMs, endMs, size); expect(result).toEqual([ { clusterUuid: clusters[0].clusterUuid, @@ -129,7 +140,7 @@ describe('fetchCpuUsageNodeStats', () => { }, }) ); - const result = await fetchCpuUsageNodeStats(esClient, clusters, index, startMs, endMs, size); + const result = await fetchCpuUsageNodeStats(esClient, clusters, startMs, endMs, size); expect(result).toEqual([ { clusterUuid: clusters[0].clusterUuid, @@ -189,7 +200,7 @@ describe('fetchCpuUsageNodeStats', () => { }, }) ); - const result = await fetchCpuUsageNodeStats(esClient, clusters, index, startMs, endMs, size); + const result = await fetchCpuUsageNodeStats(esClient, clusters, startMs, endMs, size); expect(result[0].ccs).toBe('foo'); }); @@ -203,9 +214,10 @@ describe('fetchCpuUsageNodeStats', () => { }); const filterQuery = '{"bool":{"should":[{"exists":{"field":"cluster_uuid"}}],"minimum_should_match":1}}'; - await fetchCpuUsageNodeStats(esClient, clusters, index, startMs, endMs, size, filterQuery); + await fetchCpuUsageNodeStats(esClient, clusters, startMs, endMs, size, filterQuery); expect(params).toStrictEqual({ - index: '.monitoring-es-*', + index: + '*:.monitoring-es-*,.monitoring-es-*,*:metrics-elasticsearch.node_stats-*,metrics-elasticsearch.node_stats-*', filter_path: ['aggregations'], body: { size: 0, @@ -213,7 +225,15 @@ describe('fetchCpuUsageNodeStats', () => { bool: { filter: [ { terms: { cluster_uuid: ['abc123'] } }, - { term: { type: 'node_stats' } }, + { + bool: { + should: [ + { term: { type: 'node_stats' } }, + { term: { 'data_stream.dataset': 'elasticsearch.node_stats' } }, + ], + minimum_should_match: 1, + }, + }, { range: { timestamp: { format: 'epoch_millis', gte: 0, lte: 0 } } }, { bool: { should: [{ exists: { field: 'cluster_uuid' } }], minimum_should_match: 1 }, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts index 2ad42870e99587..99fc9db7d097bb 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts @@ -10,6 +10,10 @@ import { get } from 'lodash'; import moment from 'moment'; import { NORMALIZED_DERIVATIVE_UNIT } from '../../../common/constants'; import { AlertCluster, AlertCpuUsageNodeStats } from '../../../common/types/alerts'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; interface NodeBucketESResponse { key: string; @@ -26,7 +30,6 @@ interface ClusterBucketESResponse { export async function fetchCpuUsageNodeStats( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, startMs: number, endMs: number, size: number, @@ -35,8 +38,15 @@ export async function fetchCpuUsageNodeStats( // Using pure MS didn't seem to work well with the date_histogram interval // but minutes does const intervalInMinutes = moment.duration(endMs - startMs).asMinutes(); + + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'node_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations'], body: { size: 0, @@ -48,11 +58,7 @@ export async function fetchCpuUsageNodeStats( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - { - term: { - type: 'node_stats', - }, - }, + createDatasetFilter('node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts index 4766400891af55..d4bbbe71e9c418 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts @@ -10,6 +10,18 @@ import { elasticsearchClientMock } from '../../../../../../src/core/server/elast import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { fetchDiskUsageNodeStats } from './fetch_disk_usage_node_stats'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchDiskUsageNodeStats', () => { const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; @@ -19,7 +31,6 @@ describe('fetchDiskUsageNodeStats', () => { clusterName: 'test-cluster', }, ]; - const index = '.monitoring-es-*'; const duration = '5m'; const size = 10; @@ -63,7 +74,7 @@ describe('fetchDiskUsageNodeStats', () => { }) ); - const result = await fetchDiskUsageNodeStats(esClient, clusters, index, duration, size); + const result = await fetchDiskUsageNodeStats(esClient, clusters, duration, size); expect(result).toEqual([ { clusterUuid: clusters[0].clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts index 2d4872c0bd8958..9116d477278549 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts @@ -8,18 +8,27 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { AlertCluster, AlertDiskUsageNodeStats } from '../../../common/types/alerts'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; export async function fetchDiskUsageNodeStats( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, duration: string, size: number, filterQuery?: string ): Promise { const clustersIds = clusters.map((cluster) => cluster.clusterUuid); + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'node_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations'], body: { size: 0, @@ -31,11 +40,7 @@ export async function fetchDiskUsageNodeStats( cluster_uuid: clustersIds, }, }, - { - term: { - type: 'node_stats', - }, - }, + createDatasetFilter('node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts index 515fa3b2442d3e..c8505a82614dbe 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts @@ -11,6 +11,18 @@ import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { fetchElasticsearchVersions } from './fetch_elasticsearch_versions'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchElasticsearchVersions', () => { const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; @@ -45,7 +57,7 @@ describe('fetchElasticsearchVersions', () => { } as estypes.SearchResponse) ); - const result = await fetchElasticsearchVersions(esClient, clusters, index, size); + const result = await fetchElasticsearchVersions(esClient, clusters, size); expect(result).toEqual([ { clusterUuid: clusters[0].clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts index 6ca2e89048df97..16c7f67851f3db 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts @@ -7,16 +7,25 @@ import { ElasticsearchClient } from 'kibana/server'; import { AlertCluster, AlertVersions } from '../../../common/types/alerts'; import { ElasticsearchSource, ElasticsearchResponse } from '../../../common/types/es'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; export async function fetchElasticsearchVersions( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, size: number, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'cluster_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: [ 'hits.hits._source.cluster_stats.nodes.versions', 'hits.hits._index', @@ -40,11 +49,7 @@ export async function fetchElasticsearchVersions( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - { - term: { - type: 'cluster_stats', - }, - }, + createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts index 9259adc63e5463..65ed31f5db5107 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts @@ -9,7 +9,10 @@ import { ElasticsearchClient } from 'kibana/server'; import { AlertCluster, IndexShardSizeStats } from '../../../common/types/alerts'; import { ElasticsearchIndexStats, ElasticsearchResponseHit } from '../../../common/types/es'; import { ESGlobPatterns, RegExPatterns } from '../../../common/es_glob_patterns'; +import { createDatasetFilter } from './create_dataset_query_filter'; import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; type TopHitType = ElasticsearchResponseHit & { _source: { index_stats?: Partial }; @@ -28,25 +31,26 @@ const gbMultiplier = 1000000000; export async function fetchIndexShardSize( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, threshold: number, shardIndexPatterns: string, size: number, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'index', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations.clusters.buckets'], body: { size: 0, query: { bool: { - must: [ - { - match: { - type: 'index_stats', - }, - }, + filter: [ + createDatasetFilter('index_stats', 'elasticsearch.index'), { range: { timestamp: { @@ -102,7 +106,7 @@ export async function fetchIndexShardSize( try { if (filterQuery) { const filterQueryObject = JSON.parse(filterQuery); - params.body.query.bool.must.push(filterQueryObject); + params.body.query.bool.filter.push(filterQueryObject); } } catch (e) { // meh diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts index f9a03bb73d5fcd..6bdecaf6f83edc 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts @@ -10,6 +10,18 @@ import { fetchKibanaVersions } from './fetch_kibana_versions'; import { elasticsearchClientMock } from '../../../../../../src/core/server/elasticsearch/client/mocks'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchKibanaVersions', () => { const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; const clusters = [ @@ -66,7 +78,7 @@ describe('fetchKibanaVersions', () => { }) ); - const result = await fetchKibanaVersions(esClient, clusters, index, size); + const result = await fetchKibanaVersions(esClient, clusters, size); expect(result).toEqual([ { clusterUuid: clusters[0].clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts index 71813f3a526dee..48465954e8afb3 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts @@ -7,6 +7,10 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { AlertCluster, AlertVersions } from '../../../common/types/alerts'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; interface ESAggResponse { key: string; @@ -15,12 +19,17 @@ interface ESAggResponse { export async function fetchKibanaVersions( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, size: number, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'kibana', + dataset: 'stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations'], body: { size: 0, @@ -32,11 +41,7 @@ export async function fetchKibanaVersions( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - { - term: { - type: 'kibana_stats', - }, - }, + createDatasetFilter('kibana_stats', 'kibana.stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts index 538e24a7642769..fbfe6ba58d5401 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts @@ -10,6 +10,18 @@ import { elasticsearchClientMock } from '../../../../../../src/core/server/elast import { elasticsearchServiceMock } from 'src/core/server/mocks'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchLicenses', () => { const clusterName = 'MyCluster'; const clusterUuid = 'clusterA'; diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts index b7bdf2fb6be724..0d21227112ecf8 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts @@ -7,15 +7,24 @@ import { ElasticsearchClient } from 'kibana/server'; import { AlertLicense, AlertCluster } from '../../../common/types/alerts'; import { ElasticsearchSource } from '../../../common/types/es'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; export async function fetchLicenses( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'cluster_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: [ 'hits.hits._source.license.*', 'hits.hits._source.cluster_uuid', @@ -39,11 +48,7 @@ export async function fetchLicenses( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - { - term: { - type: 'cluster_stats', - }, - }, + createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts index 5732fc00f009b5..a1df1a56ad3b54 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts @@ -10,6 +10,18 @@ import { fetchLogstashVersions } from './fetch_logstash_versions'; import { elasticsearchClientMock } from '../../../../../../src/core/server/elasticsearch/client/mocks'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('fetchLogstashVersions', () => { const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; const clusters = [ @@ -66,7 +78,7 @@ describe('fetchLogstashVersions', () => { }) ); - const result = await fetchLogstashVersions(esClient, clusters, index, size); + const result = await fetchLogstashVersions(esClient, clusters, size); expect(result).toEqual([ { clusterUuid: clusters[0].clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts index 112c2fe798b100..3527223c3b07a3 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts @@ -7,6 +7,10 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { AlertCluster, AlertVersions } from '../../../common/types/alerts'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; interface ESAggResponse { key: string; @@ -15,12 +19,17 @@ interface ESAggResponse { export async function fetchLogstashVersions( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, size: number, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'logstash', + dataset: 'node_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations'], body: { size: 0, @@ -32,11 +41,7 @@ export async function fetchLogstashVersions( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - { - term: { - type: 'logstash_stats', - }, - }, + createDatasetFilter('logstash_stats', 'logstash.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts index 9403ae5d79a70f..96462462ada6b7 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts @@ -8,19 +8,28 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { AlertCluster, AlertMemoryUsageNodeStats } from '../../../common/types/alerts'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; export async function fetchMemoryUsageNodeStats( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, startMs: number, endMs: number, size: number, filterQuery?: string ): Promise { const clustersIds = clusters.map((cluster) => cluster.clusterUuid); + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'node_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations'], body: { size: 0, @@ -32,11 +41,7 @@ export async function fetchMemoryUsageNodeStats( cluster_uuid: clustersIds, }, }, - { - term: { - type: 'node_stats', - }, - }, + createDatasetFilter('node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts index 980adb009ff8f6..a51cced18fd7ba 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts @@ -9,6 +9,18 @@ import { elasticsearchClientMock } from '../../../../../../src/core/server/elasticsearch/client/mocks'; import { fetchMissingMonitoringData } from './fetch_missing_monitoring_data'; +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + function getResponse( index: string, products: Array<{ @@ -42,7 +54,6 @@ function getResponse( describe('fetchMissingMonitoringData', () => { const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - const index = '.monitoring-*'; const startMs = 100; const size = 10; @@ -87,7 +98,7 @@ describe('fetchMissingMonitoringData', () => { }, }) ); - const result = await fetchMissingMonitoringData(esClient, clusters, index, size, now, startMs); + const result = await fetchMissingMonitoringData(esClient, clusters, size, now, startMs); expect(result).toEqual([ { nodeId: 'nodeUuid1', @@ -137,7 +148,7 @@ describe('fetchMissingMonitoringData', () => { }, }) ); - const result = await fetchMissingMonitoringData(esClient, clusters, index, size, now, startMs); + const result = await fetchMissingMonitoringData(esClient, clusters, size, now, startMs); expect(result).toEqual([ { nodeId: 'nodeUuid1', diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts index cdf0f21b52b09d..93ad44a5fd44bf 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts @@ -8,6 +8,9 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { AlertCluster, AlertMissingData } from '../../../common/types/alerts'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; interface ClusterBucketESResponse { key: string; @@ -44,15 +47,21 @@ interface TopHitESResponse { export async function fetchMissingMonitoringData( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, size: number, nowInMs: number, startMs: number, filterQuery?: string ): Promise { const endMs = nowInMs; + // changing this to only search ES because of changes related to https://github.com/elastic/kibana/issues/83309 + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'node_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations.clusters.buckets'], body: { size: 0, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts index 3dc3e315318fcb..b67d071afcc0df 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts @@ -7,6 +7,10 @@ import { ElasticsearchClient } from 'kibana/server'; import { AlertCluster, AlertClusterStatsNodes } from '../../../common/types/alerts'; import { ElasticsearchSource } from '../../../common/types/es'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; function formatNode( nodes: NonNullable['nodes']> | undefined @@ -26,11 +30,16 @@ function formatNode( export async function fetchNodesFromClusterStats( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, filterQuery?: string ): Promise { + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'cluster_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations.clusters.buckets'], body: { size: 0, @@ -45,11 +54,7 @@ export async function fetchNodesFromClusterStats( query: { bool: { filter: [ - { - term: { - type: 'cluster_stats', - }, - }, + createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts index 0d1d052b5f8663..472bf0e955a2d2 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts @@ -8,6 +8,10 @@ import { ElasticsearchClient } from 'kibana/server'; import { get } from 'lodash'; import { AlertCluster, AlertThreadPoolRejectionsStats } from '../../../common/types/alerts'; +import { createDatasetFilter } from './create_dataset_query_filter'; +import { Globals } from '../../static_globals'; +import { getConfigCcs } from '../../../common/ccs_utils'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; const invalidNumberValue = (value: number) => { return isNaN(value) || value === undefined || value === null; @@ -33,15 +37,20 @@ const getTopHits = (threadType: string, order: 'asc' | 'desc') => ({ export async function fetchThreadPoolRejectionStats( esClient: ElasticsearchClient, clusters: AlertCluster[], - index: string, size: number, threadType: string, duration: string, filterQuery?: string ): Promise { const clustersIds = clusters.map((cluster) => cluster.clusterUuid); + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + dataset: 'node_stats', + ccs: getConfigCcs(Globals.app.config) ? '*' : undefined, + }); const params = { - index, + index: indexPatterns, filter_path: ['aggregations'], body: { size: 0, @@ -53,11 +62,7 @@ export async function fetchThreadPoolRejectionStats( cluster_uuid: clustersIds, }, }, - { - term: { - type: 'node_stats', - }, - }, + createDatasetFilter('node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/apm/create_apm_query.ts b/x-pack/plugins/monitoring/server/lib/apm/create_apm_query.ts index 63c56607a68e6b..1dc0f7a31e9a7c 100644 --- a/x-pack/plugins/monitoring/server/lib/apm/create_apm_query.ts +++ b/x-pack/plugins/monitoring/server/lib/apm/create_apm_query.ts @@ -25,7 +25,9 @@ export function createApmQuery(options: { const opts = { filters: [] as any[], metric: ApmMetric.getMetricFields(), - types: ['stats', 'beats_stats'], + type: 'beats_stats', + metricset: 'stats', + dsDataset: 'beats.stats', ...(options ?? {}), }; @@ -38,6 +40,5 @@ export function createApmQuery(options: { }, }, }); - return createQuery(opts); } diff --git a/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.ts b/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.ts index e99ce0da1ef102..77b5e447549990 100644 --- a/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.ts @@ -6,12 +6,13 @@ */ import { LegacyRequest, Cluster } from '../../types'; -import { checkParam } from '../error_missing_required'; import { createApmQuery } from './create_apm_query'; import { ApmMetric } from '../metrics'; import { apmAggResponseHandler, apmUuidsAgg, apmAggFilterPath } from './_apm_stats'; import { getTimeOfLastEvent } from './_get_time_of_last_event'; import { ElasticsearchResponse } from '../../../common/types/es'; +import { getLegacyIndexPattern } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; export function handleResponse(clusterUuid: string, response: ElasticsearchResponse) { const { apmTotal, totalEvents, memRss, versions } = apmAggResponseHandler(response); @@ -32,24 +33,24 @@ export function handleResponse(clusterUuid: string, response: ElasticsearchRespo }; } -export function getApmsForClusters( - req: LegacyRequest, - apmIndexPattern: string, - clusters: Cluster[] -) { - checkParam(apmIndexPattern, 'apmIndexPattern in apms/getApmsForClusters'); - +export function getApmsForClusters(req: LegacyRequest, clusters: Cluster[], ccs?: string) { const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; const config = req.server.config(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const cgroup = config.get('monitoring.ui.container.apm.enabled'); + const indexPatterns = getLegacyIndexPattern({ + moduleType: 'beats', + ccs: ccs || req.payload.ccs, + config: Globals.app.config, + }); + return Promise.all( clusters.map(async (cluster) => { const clusterUuid = cluster.elasticsearch?.cluster?.id ?? cluster.cluster_uuid; const params = { - index: apmIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, filter_path: apmAggFilterPath, @@ -70,7 +71,7 @@ export function getApmsForClusters( getTimeOfLastEvent({ req, callWithRequest, - apmIndexPattern, + apmIndexPattern: indexPatterns, start, end, clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/beats/create_beats_query.ts b/x-pack/plugins/monitoring/server/lib/beats/create_beats_query.ts index b013cd8234c406..6f56a95c8cbb95 100644 --- a/x-pack/plugins/monitoring/server/lib/beats/create_beats_query.ts +++ b/x-pack/plugins/monitoring/server/lib/beats/create_beats_query.ts @@ -26,9 +26,12 @@ export function createBeatsQuery(options: { end?: number; }) { const opts = { + moduleType: 'beats', filters: [] as any[], metric: BeatsMetric.getMetricFields(), - types: ['stats', 'beats_stats'], + type: 'beats_stats', + metricset: 'stats', + dsDataset: 'beats.stats', ...(options ?? {}), }; diff --git a/x-pack/plugins/monitoring/server/lib/beats/get_beats_for_clusters.ts b/x-pack/plugins/monitoring/server/lib/beats/get_beats_for_clusters.ts index 3a0720f7ca1956..ae723fa99bbded 100644 --- a/x-pack/plugins/monitoring/server/lib/beats/get_beats_for_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/beats/get_beats_for_clusters.ts @@ -5,12 +5,13 @@ * 2.0. */ -import { checkParam } from '../error_missing_required'; import { BeatsClusterMetric } from '../metrics'; import { createBeatsQuery } from './create_beats_query'; import { beatsAggFilterPath, beatsUuidsAgg, beatsAggResponseHandler } from './_beats_stats'; import type { ElasticsearchResponse } from '../../../common/types/es'; import { LegacyRequest, Cluster } from '../../types'; +import { getLegacyIndexPattern } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; export function handleResponse(clusterUuid: string, response: ElasticsearchResponse) { const { beatTotal, beatTypes, totalEvents, bytesSent } = beatsAggResponseHandler(response); @@ -31,23 +32,21 @@ export function handleResponse(clusterUuid: string, response: ElasticsearchRespo }; } -export function getBeatsForClusters( - req: LegacyRequest, - beatsIndexPattern: string, - clusters: Cluster[] -) { - checkParam(beatsIndexPattern, 'beatsIndexPattern in beats/getBeatsForClusters'); - +export function getBeatsForClusters(req: LegacyRequest, clusters: Cluster[], ccs: string) { const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; const config = req.server.config(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); - + const indexPatterns = getLegacyIndexPattern({ + moduleType: 'beats', + ccs, + config: Globals.app.config, + }); return Promise.all( clusters.map(async (cluster) => { const clusterUuid = cluster.elasticsearch?.cluster?.id ?? cluster.cluster_uuid; const params = { - index: beatsIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, filter_path: beatsAggFilterPath, diff --git a/x-pack/plugins/monitoring/server/lib/cluster/flag_supported_clusters.ts b/x-pack/plugins/monitoring/server/lib/cluster/flag_supported_clusters.ts index 820b1bf24c6a13..9ead8833dce462 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/flag_supported_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/cluster/flag_supported_clusters.ts @@ -10,15 +10,24 @@ import { checkParam } from '../error_missing_required'; import { STANDALONE_CLUSTER_CLUSTER_UUID } from '../../../common/constants'; import { ElasticsearchResponse, ElasticsearchModifiedSource } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from './get_index_patterns'; +import { Globals } from '../../static_globals'; async function findSupportedBasicLicenseCluster( req: LegacyRequest, clusters: ElasticsearchModifiedSource[], - kbnIndexPattern: string, + ccs: string, kibanaUuid: string, serverLog: (message: string) => void ) { - checkParam(kbnIndexPattern, 'kbnIndexPattern in cluster/findSupportedBasicLicenseCluster'); + const dataset = 'stats'; + const moduleType = 'kibana'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + ccs, + }); serverLog( `Detected all clusters in monitoring data have basic license. Checking for supported admin cluster UUID for Kibana ${kibanaUuid}.` @@ -28,7 +37,7 @@ async function findSupportedBasicLicenseCluster( const gte = req.payload.timeRange.min; const lte = req.payload.timeRange.max; const kibanaDataResult: ElasticsearchResponse = (await callWithRequest(req, 'search', { - index: kbnIndexPattern, + index: indexPatterns, size: 1, ignore_unavailable: true, filter_path: ['hits.hits._source.cluster_uuid', 'hits.hits._source.cluster.id'], @@ -41,7 +50,7 @@ async function findSupportedBasicLicenseCluster( bool: { should: [ { term: { type: 'kibana_stats' } }, - { term: { 'metricset.name': 'stats' } }, + { term: { 'data_stream.dataset': 'kibana.stats' } }, ], }, }, @@ -58,7 +67,6 @@ async function findSupportedBasicLicenseCluster( cluster.isSupported = true; } } - serverLog( `Found basic license admin cluster UUID for Monitoring UI support: ${supportedClusterUuid}.` ); @@ -80,9 +88,7 @@ async function findSupportedBasicLicenseCluster( * Non-Basic license clusters and any cluster in a single-cluster environment * are also flagged as supported in this method. */ -export function flagSupportedClusters(req: LegacyRequest, kbnIndexPattern: string) { - checkParam(kbnIndexPattern, 'kbnIndexPattern in cluster/flagSupportedClusters'); - +export function flagSupportedClusters(req: LegacyRequest, ccs: string) { const config = req.server.config(); const serverLog = (message: string) => req.getLogger('supported-clusters').debug(message); const flagAllSupported = (clusters: ElasticsearchModifiedSource[]) => { @@ -124,13 +130,7 @@ export function flagSupportedClusters(req: LegacyRequest, kbnIndexPattern: strin // if all linked are basic licenses if (linkedClusterCount === basicLicenseCount) { const kibanaUuid = config.get('server.uuid') as string; - return await findSupportedBasicLicenseCluster( - req, - clusters, - kbnIndexPattern, - kibanaUuid, - serverLog - ); + return await findSupportedBasicLicenseCluster(req, clusters, ccs, kibanaUuid, serverLog); } // if some non-basic licenses diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_license.ts b/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_license.ts index 8ed5578e574a0c..108e88529bf7c7 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_license.ts +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_license.ts @@ -13,19 +13,31 @@ import { createQuery } from '../create_query'; import { ElasticsearchMetric } from '../metrics'; import { ElasticsearchResponse } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from './get_index_patterns'; +import { Globals } from '../../static_globals'; -export function getClusterLicense(req: LegacyRequest, esIndexPattern: string, clusterUuid: string) { - checkParam(esIndexPattern, 'esIndexPattern in getClusterLicense'); +// is this being used anywhere? not called within the app +export function getClusterLicense(req: LegacyRequest, clusterUuid: string) { + const dataset = 'cluster_stats'; + const moduleType = 'elasticsearch'; + const indexPattern = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + ccs: req.payload.ccs, + }); const params = { - index: esIndexPattern, + index: indexPattern, size: 1, ignore_unavailable: true, filter_path: ['hits.hits._source.license'], body: { sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, query: createQuery({ - type: 'cluster_stats', + type: dataset, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, clusterUuid, metric: ElasticsearchMetric.getMetricFields(), }), diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_stats.ts b/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_stats.ts index c671765a445486..97eebd678a975e 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_cluster_stats.ts @@ -14,11 +14,10 @@ import { getClustersStats } from './get_clusters_stats'; * This will fetch the cluster stats and cluster state as a single object for the cluster specified by the {@code req}. * * @param {Object} req The incoming user's request - * @param {String} esIndexPattern The Elasticsearch index pattern * @param {String} clusterUuid The requested cluster's UUID * @return {Promise} The object cluster response. */ -export function getClusterStats(req: LegacyRequest, esIndexPattern: string, clusterUuid: string) { +export function getClusterStats(req: LegacyRequest, clusterUuid: string) { if (!clusterUuid) { throw badRequest( i18n.translate('xpack.monitoring.clusterStats.uuidNotSpecifiedErrorMessage', { @@ -29,7 +28,7 @@ export function getClusterStats(req: LegacyRequest, esIndexPattern: string, clus } // passing clusterUuid so `get_clusters` will filter for single cluster - return getClustersStats(req, esIndexPattern, clusterUuid).then((clusters) => { + return getClustersStats(req, clusterUuid).then((clusters) => { if (!clusters || clusters.length === 0) { throw notFound( i18n.translate('xpack.monitoring.clusterStats.uuidNotFoundErrorMessage', { diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.ts b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.ts index 6e28070cdbfaaf..570d6fadfeb903 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.ts +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.ts @@ -52,15 +52,7 @@ export async function getClustersFromRequest( codePaths, }: { clusterUuid: string; start: number; end: number; codePaths: string[] } ) { - const { - esIndexPattern, - kbnIndexPattern, - lsIndexPattern, - beatsIndexPattern, - apmIndexPattern, - enterpriseSearchIndexPattern, - filebeatIndexPattern, - } = indexPatterns; + const { filebeatIndexPattern } = indexPatterns; const config = req.server.config(); const isStandaloneCluster = clusterUuid === STANDALONE_CLUSTER_CLUSTER_UUID; @@ -71,18 +63,11 @@ export async function getClustersFromRequest( clusters.push(getStandaloneClusterDefinition()); } else { // get clusters with stats and cluster state - clusters = await getClustersStats(req, esIndexPattern, clusterUuid); + clusters = await getClustersStats(req, clusterUuid, '*'); } if (!clusterUuid && !isStandaloneCluster) { - const indexPatternsToCheckForNonClusters = [ - lsIndexPattern, - beatsIndexPattern, - apmIndexPattern, - enterpriseSearchIndexPattern, - ]; - - if (await hasStandaloneClusters(req, indexPatternsToCheckForNonClusters)) { + if (await hasStandaloneClusters(req, '*')) { clusters.push(getStandaloneClusterDefinition()); } } @@ -106,7 +91,7 @@ export async function getClustersFromRequest( // add ml jobs and alerts data const mlJobs = isInCodePath(codePaths, [CODE_PATH_ML]) - ? await getMlJobsForCluster(req, esIndexPattern, cluster) + ? await getMlJobsForCluster(req, cluster, '*') : null; if (mlJobs !== null) { cluster.ml = { jobs: mlJobs }; @@ -128,7 +113,7 @@ export async function getClustersFromRequest( } // update clusters with license check results - const getSupportedClusters = flagSupportedClusters(req, kbnIndexPattern); + const getSupportedClusters = flagSupportedClusters(req, '*'); clusters = await getSupportedClusters(clusters); // add alerts data @@ -184,7 +169,7 @@ export async function getClustersFromRequest( // add kibana data const kibanas = isInCodePath(codePaths, [CODE_PATH_KIBANA]) && !isStandaloneCluster - ? await getKibanasForClusters(req, kbnIndexPattern, clusters) + ? await getKibanasForClusters(req, clusters, '*') : []; // add the kibana data to each cluster kibanas.forEach((kibana) => { @@ -197,8 +182,8 @@ export async function getClustersFromRequest( // add logstash data if (isInCodePath(codePaths, [CODE_PATH_LOGSTASH])) { - const logstashes = await getLogstashForClusters(req, lsIndexPattern, clusters); - const pipelines = await getLogstashPipelineIds({ req, lsIndexPattern, clusterUuid, size: 1 }); + const logstashes = await getLogstashForClusters(req, clusters, '*'); + const pipelines = await getLogstashPipelineIds({ req, clusterUuid, size: 1, ccs: '*' }); logstashes.forEach((logstash) => { const clusterIndex = clusters.findIndex( (cluster) => @@ -214,7 +199,7 @@ export async function getClustersFromRequest( // add beats data const beatsByCluster = isInCodePath(codePaths, [CODE_PATH_BEATS]) - ? await getBeatsForClusters(req, beatsIndexPattern, clusters) + ? await getBeatsForClusters(req, clusters, '*') : []; beatsByCluster.forEach((beats) => { const clusterIndex = clusters.findIndex( @@ -226,7 +211,7 @@ export async function getClustersFromRequest( // add apm data const apmsByCluster = isInCodePath(codePaths, [CODE_PATH_APM]) - ? await getApmsForClusters(req, apmIndexPattern, clusters) + ? await getApmsForClusters(req, clusters, '*') : []; apmsByCluster.forEach((apm) => { const clusterIndex = clusters.findIndex( @@ -244,7 +229,7 @@ export async function getClustersFromRequest( // add Enterprise Search data const enterpriseSearchByCluster = isInCodePath(codePaths, [CODE_PATH_ENTERPRISE_SEARCH]) - ? await getEnterpriseSearchForClusters(req, enterpriseSearchIndexPattern, clusters) + ? await getEnterpriseSearchForClusters(req, clusters, '*') : []; enterpriseSearchByCluster.forEach((entSearch) => { const clusterIndex = clusters.findIndex( @@ -259,7 +244,7 @@ export async function getClustersFromRequest( }); // check ccr configuration - const isCcrEnabled = await checkCcrEnabled(req, esIndexPattern); + const isCcrEnabled = await checkCcrEnabled(req, '*'); const kibanaUuid = config.get('server.uuid')!; diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_state.ts b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_state.ts index d732b43bc203b4..fab74ef1d979fd 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_state.ts +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_state.ts @@ -10,6 +10,8 @@ import { find } from 'lodash'; import { checkParam } from '../error_missing_required'; import { ElasticsearchResponse, ElasticsearchModifiedSource } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from './get_index_patterns'; +import { Globals } from '../../static_globals'; /** * Augment the {@clusters} with their cluster state's from the {@code response}. @@ -46,13 +48,7 @@ export function handleResponse( * * If there is no cluster state available for any cluster, then it will be returned without any cluster state information. */ -export function getClustersState( - req: LegacyRequest, - esIndexPattern: string, - clusters: ElasticsearchModifiedSource[] -) { - checkParam(esIndexPattern, 'esIndexPattern in cluster/getClustersHealth'); - +export function getClustersState(req: LegacyRequest, clusters: ElasticsearchModifiedSource[]) { const clusterUuids = clusters .filter((cluster) => !cluster.cluster_state || !cluster.elasticsearch?.cluster?.stats?.state) .map((cluster) => cluster.cluster_uuid || cluster.elasticsearch?.cluster?.id); @@ -63,8 +59,14 @@ export function getClustersState( return Promise.resolve(clusters); } + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'elasticsearch', + ccs: req.payload.ccs, + }); + const params = { - index: esIndexPattern, + index: indexPatterns, size: clusterUuids.length, ignore_unavailable: true, filter_path: [ diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.test.js b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.test.js index a8f7423c45100c..db4339a23e88c9 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.test.js +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.test.js @@ -43,7 +43,7 @@ describe('handleClusterStats', () => { }, }; - const clusters = handleClusterStats(response, { log: () => undefined }); + const clusters = handleClusterStats(response); expect(clusters.length).toEqual(1); expect(clusters[0].ccs).toEqual('cluster_one'); diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.ts b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.ts index a2201ca958e355..80fe5ce3806a67 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.ts @@ -16,21 +16,22 @@ import { parseCrossClusterPrefix } from '../../../common/ccs_utils'; import { getClustersState } from './get_clusters_state'; import { ElasticsearchResponse, ElasticsearchModifiedSource } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from './get_index_patterns'; +import { Globals } from '../../static_globals'; /** * This will fetch the cluster stats and cluster state as a single object per cluster. * * @param {Object} req The incoming user's request - * @param {String} esIndexPattern The Elasticsearch index pattern * @param {String} clusterUuid (optional) If not undefined, getClusters will filter for a single cluster * @return {Promise} A promise containing an array of clusters. */ -export function getClustersStats(req: LegacyRequest, esIndexPattern: string, clusterUuid: string) { +export function getClustersStats(req: LegacyRequest, clusterUuid: string, ccs?: string) { return ( - fetchClusterStats(req, esIndexPattern, clusterUuid) + fetchClusterStats(req, clusterUuid, ccs) .then((response) => handleClusterStats(response)) // augment older documents (e.g., from 2.x - 5.4) with their cluster_state - .then((clusters) => getClustersState(req, esIndexPattern, clusters)) + .then((clusters) => getClustersState(req, clusters)) ); } @@ -38,12 +39,19 @@ export function getClustersStats(req: LegacyRequest, esIndexPattern: string, clu * Query cluster_stats for all the cluster data * * @param {Object} req (required) - server request - * @param {String} esIndexPattern (required) - index pattern to use in searching for cluster_stats data * @param {String} clusterUuid (optional) - if not undefined, getClusters filters for a single clusterUuid * @return {Promise} Object representing each cluster. */ -function fetchClusterStats(req: LegacyRequest, esIndexPattern: string, clusterUuid: string) { - checkParam(esIndexPattern, 'esIndexPattern in getClusters'); +function fetchClusterStats(req: LegacyRequest, clusterUuid: string, ccs?: string) { + const dataset = 'cluster_stats'; + const moduleType = 'elasticsearch'; + const indexPattern = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + // this is will be either *, a request value, or null + ccs: ccs || req.payload.ccs, + }); const config = req.server.config(); // Get the params from the POST body for the request @@ -51,7 +59,7 @@ function fetchClusterStats(req: LegacyRequest, esIndexPattern: string, clusterUu const end = req.payload.timeRange.max; const metric = ElasticsearchMetric.getMetricFields(); const params = { - index: esIndexPattern, + index: indexPattern, size: config.get('monitoring.ui.max_bucket_size'), ignore_unavailable: true, filter_path: [ @@ -80,7 +88,15 @@ function fetchClusterStats(req: LegacyRequest, esIndexPattern: string, clusterUu 'hits.hits._source.cluster_settings.cluster.metadata.display_name', ], body: { - query: createQuery({ type: 'cluster_stats', start, end, metric, clusterUuid }), + query: createQuery({ + type: dataset, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, + start, + end, + metric, + clusterUuid, + }), collapse: { field: 'cluster_uuid', }, diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.test.ts b/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.test.ts new file mode 100644 index 00000000000000..015bdd4c65ccfe --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.test.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MonitoringConfig } from '../..'; +import { getNewIndexPatterns } from './get_index_patterns'; + +const getConfigWithCcs = (ccsEnabled: boolean) => { + return { + ui: { + ccs: { + enabled: ccsEnabled, + }, + }, + } as MonitoringConfig; +}; + +describe('getNewIndexPatterns', () => { + beforeEach(() => { + jest.resetModules(); + }); + it('returns local elasticsearch index patterns when ccs is enabled (default true) and no ccs payload', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + moduleType: 'elasticsearch', + }); + expect(indexPatterns).toBe('.monitoring-es-*,metrics-elasticsearch.*-*'); + }); + it('returns ecs only elasticsearch index patterns when specifying ecsLegacyOnly: true', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + moduleType: 'elasticsearch', + ecsLegacyOnly: true, + }); + expect(indexPatterns).toBe('.monitoring-es-8-*,metrics-elasticsearch.*-*'); + }); + it('returns local kibana index patterns when ccs is enabled with no ccs payload', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + moduleType: 'kibana', + }); + expect(indexPatterns).toBe('.monitoring-kibana-*,metrics-kibana.*-*'); + }); + it('returns logstash index patterns when ccs is enabled and no ccs payload', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + moduleType: 'logstash', + }); + expect(indexPatterns).toBe('.monitoring-logstash-*,metrics-logstash.*-*'); + }); + it('returns beats index patterns when ccs is enabled and no ccs payload', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + moduleType: 'beats', + }); + expect(indexPatterns).toBe('.monitoring-beats-*,metrics-beats.*-*'); + }); + it('returns elasticsearch index patterns with dataset', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + moduleType: 'elasticsearch', + dataset: 'cluster_stats', + }); + expect(indexPatterns).toBe('.monitoring-es-*,metrics-elasticsearch.cluster_stats-*'); + }); + it('returns elasticsearch index patterns without ccs prefixes when ccs is disabled', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(false), + moduleType: 'elasticsearch', + }); + expect(indexPatterns).toBe('.monitoring-es-*,metrics-elasticsearch.*-*'); + }); + it('returns elasticsearch index patterns without ccs prefixes when ccs is disabled but ccs request payload has a value', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(false), + ccs: 'myccs', + moduleType: 'elasticsearch', + }); + expect(indexPatterns).toBe('.monitoring-es-*,metrics-elasticsearch.*-*'); + }); + it('returns elasticsearch index patterns with custom ccs prefixes when ccs is enabled and ccs request payload has a value', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + ccs: 'myccs', + moduleType: 'elasticsearch', + }); + expect(indexPatterns).toBe('myccs:.monitoring-es-*,myccs:metrics-elasticsearch.*-*'); + }); + it('returns elasticsearch index patterns with ccs prefixes and local index patterns when ccs is enabled and ccs request payload value is *', () => { + const indexPatterns = getNewIndexPatterns({ + config: getConfigWithCcs(true), + ccs: '*', + moduleType: 'elasticsearch', + }); + expect(indexPatterns).toBe( + '*:.monitoring-es-*,.monitoring-es-*,*:metrics-elasticsearch.*-*,metrics-elasticsearch.*-*' + ); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.ts b/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.ts index 5f7e55cf94c5a5..fe9a319780db5b 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.ts +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.ts @@ -9,12 +9,17 @@ import { LegacyServer } from '../../types'; import { prefixIndexPattern } from '../../../common/ccs_utils'; import { INDEX_PATTERN_ELASTICSEARCH, + INDEX_PATTERN_ELASTICSEARCH_ECS, INDEX_PATTERN_KIBANA, INDEX_PATTERN_LOGSTASH, INDEX_PATTERN_BEATS, INDEX_ALERTS, + DS_INDEX_PATTERN_TYPES, + DS_INDEX_PATTERN_METRICS, + INDEX_PATTERN_TYPES, INDEX_PATTERN_ENTERPRISE_SEARCH, } from '../../../common/constants'; +import { MonitoringConfig } from '../..'; export function getIndexPatterns( server: LegacyServer, @@ -44,10 +49,90 @@ export function getIndexPatterns( ...Object.keys(additionalPatterns).reduce((accum, varName) => { return { ...accum, - [varName]: prefixIndexPattern(config, additionalPatterns[varName], ccs, true), + [varName]: prefixIndexPattern(config, additionalPatterns[varName], ccs), }; }, {}), }; - return indexPatterns; } +// calling legacy index patterns those that are .monitoring +export function getLegacyIndexPattern({ + moduleType, + ecsLegacyOnly = false, + config, + ccs, +}: { + moduleType: INDEX_PATTERN_TYPES; + ecsLegacyOnly?: boolean; + config: MonitoringConfig; + ccs?: string; +}) { + let indexPattern = ''; + switch (moduleType) { + case 'elasticsearch': + // there may be cases where we only want the legacy ecs version index pattern (>=8.0) + indexPattern = ecsLegacyOnly ? INDEX_PATTERN_ELASTICSEARCH_ECS : INDEX_PATTERN_ELASTICSEARCH; + break; + case 'kibana': + indexPattern = INDEX_PATTERN_KIBANA; + break; + case 'logstash': + indexPattern = INDEX_PATTERN_LOGSTASH; + break; + case 'beats': + indexPattern = INDEX_PATTERN_BEATS; + break; + case 'enterprisesearch': + indexPattern = INDEX_PATTERN_ENTERPRISE_SEARCH; + break; + default: + throw new Error(`invalid module type to create index pattern: ${moduleType}`); + } + return prefixIndexPattern(config, indexPattern, ccs); +} + +export function getDsIndexPattern({ + type = DS_INDEX_PATTERN_METRICS, + moduleType, + dataset, + namespace = '*', + config, + ccs, +}: { + type?: string; + dataset?: string; + moduleType: INDEX_PATTERN_TYPES; + namespace?: string; + config: MonitoringConfig; + ccs?: string; +}): string { + let datasetsPattern = ''; + if (dataset) { + datasetsPattern = `${moduleType}.${dataset}`; + } else { + datasetsPattern = `${moduleType}.*`; + } + return prefixIndexPattern(config, `${type}-${datasetsPattern}-${namespace}`, ccs); +} + +export function getNewIndexPatterns({ + config, + moduleType, + type = DS_INDEX_PATTERN_METRICS, + dataset, + namespace = '*', + ccs, + ecsLegacyOnly, +}: { + config: MonitoringConfig; + moduleType: INDEX_PATTERN_TYPES; + type?: DS_INDEX_PATTERN_TYPES; + dataset?: string; + namespace?: string; + ccs?: string; + ecsLegacyOnly?: boolean; +}): string { + const legacyIndexPattern = getLegacyIndexPattern({ moduleType, ecsLegacyOnly, config, ccs }); + const dsIndexPattern = getDsIndexPattern({ type, moduleType, dataset, namespace, config, ccs }); + return `${legacyIndexPattern},${dsIndexPattern}`; +} diff --git a/x-pack/plugins/monitoring/server/lib/create_query.test.js b/x-pack/plugins/monitoring/server/lib/create_query.test.js deleted file mode 100644 index 60fa6faa79e443..00000000000000 --- a/x-pack/plugins/monitoring/server/lib/create_query.test.js +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { set } from '@elastic/safer-lodash-set'; -import { MissingRequiredError } from './error_missing_required'; -import { ElasticsearchMetric } from './metrics'; -import { createQuery } from './create_query'; - -let metric; - -describe('Create Query', () => { - beforeEach(() => { - metric = ElasticsearchMetric.getMetricFields(); - }); - - it('Allows UUID to not be passed', () => { - const options = { metric }; - const result = createQuery(options); - const expected = set({}, 'bool.filter', []); - expect(result).toEqual(expected); - }); - - it('Uses Elasticsearch timestamp field for start and end time range by default', () => { - const options = { - uuid: 'abc123', - start: '2016-03-01 10:00:00', - end: '2016-03-01 10:00:01', - metric, - }; - const result = createQuery(options); - let expected = {}; - expected = set(expected, 'bool.filter[0].term', { - 'source_node.uuid': 'abc123', - }); - expected = set(expected, 'bool.filter[1].range.timestamp', { - format: 'epoch_millis', - gte: 1456826400000, - lte: 1456826401000, - }); - expect(result).toEqual(expected); - }); - - it('Injects uuid and timestamp fields dynamically, based on metric', () => { - const options = { - uuid: 'abc123', - start: '2016-03-01 10:00:00', - end: '2016-03-01 10:00:01', - metric: { - uuidField: 'testUuidField', - timestampField: 'testTimestampField', - }, - }; - const result = createQuery(options); - let expected = set({}, 'bool.filter[0].term.testUuidField', 'abc123'); - expected = set(expected, 'bool.filter[1].range.testTimestampField', { - format: 'epoch_millis', - gte: 1456826400000, - lte: 1456826401000, - }); - expect(result).toEqual(expected); - }); - - it('Throws if missing metric.timestampField', () => { - function callCreateQuery() { - const options = {}; // missing metric object - return createQuery(options); - } - expect(callCreateQuery).toThrowError(MissingRequiredError); - }); - - it('Throws if given uuid but missing metric.uuidField', () => { - function callCreateQuery() { - const options = { uuid: 'abc123', metric }; - delete options.metric.uuidField; - return createQuery(options); - } - expect(callCreateQuery).toThrowError(MissingRequiredError); - }); - - // TODO: tests were not running and need to be updated to pass - it.skip('Uses `type` option to add type filter with minimal fields', () => { - const options = { type: 'test-type-yay', metric }; - const result = createQuery(options); - let expected = {}; - expected = set(expected, 'bool.filter[0].term', { type: 'test-type-yay' }); - expect(result).to.be.eql(expected); - }); - - it.skip('Uses `type` option to add type filter with all other option fields', () => { - const options = { - type: 'test-type-yay', - uuid: 'abc123', - start: '2016-03-01 10:00:00', - end: '2016-03-01 10:00:01', - metric, - }; - const result = createQuery(options); - let expected = {}; - expected = set(expected, 'bool.filter[0].term', { type: 'test-type-yay' }); - expected = set(expected, 'bool.filter[1].term', { - 'source_node.uuid': 'abc123', - }); - expected = set(expected, 'bool.filter[2].range.timestamp', { - format: 'epoch_millis', - gte: 1456826400000, - lte: 1456826401000, - }); - expect(result).to.be.eql(expected); - }); -}); diff --git a/x-pack/plugins/monitoring/server/lib/create_query.test.ts b/x-pack/plugins/monitoring/server/lib/create_query.test.ts new file mode 100644 index 00000000000000..0e78889ca71fa7 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/create_query.test.ts @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MissingRequiredError } from './error_missing_required'; +import { ElasticsearchMetric } from './metrics'; +import { createQuery } from './create_query'; + +interface Metric { + uuidField?: string; + timestampField: string; +} +let metric: Metric; + +describe('Create Query', () => { + beforeEach(() => { + metric = ElasticsearchMetric.getMetricFields(); + }); + + it('Allows UUID to not be passed', () => { + const options = { metric, clusterUuid: 'cuid123' }; + expect(createQuery(options)).toEqual({ + bool: { filter: [{ term: { cluster_uuid: 'cuid123' } }] }, + }); + }); + + it('Uses Elasticsearch timestamp field for start and end time range by default', () => { + const options = { + clusterUuid: 'cuid123', + uuid: 'abc123', + start: 1456826400000, + end: 14568264010000, + metric, + }; + expect(createQuery(options)).toEqual({ + bool: { + filter: [ + { term: { cluster_uuid: 'cuid123' } }, + { term: { 'source_node.uuid': 'abc123' } }, + { + range: { + timestamp: { format: 'epoch_millis', gte: 1456826400000, lte: 14568264010000 }, + }, + }, + ], + }, + }); + }); + + it('Injects uuid and timestamp fields dynamically, based on metric', () => { + const options = { + clusterUuid: 'cuid123', + uuid: 'abc123', + start: 1456826400000, + end: 14568264010000, + metric: { + uuidField: 'testUuidField', + timestampField: 'testTimestampField', + }, + }; + expect(createQuery(options)).toEqual({ + bool: { + filter: [ + { term: { cluster_uuid: 'cuid123' } }, + { term: { testUuidField: 'abc123' } }, + { + range: { + testTimestampField: { + format: 'epoch_millis', + gte: 1456826400000, + lte: 14568264010000, + }, + }, + }, + ], + }, + }); + }); + + it('Throws if missing metric.timestampField', () => { + function callCreateQuery() { + const options = { clusterUuid: 'cuid123' }; // missing metric object + return createQuery(options); + } + expect(callCreateQuery).toThrowError(MissingRequiredError); + }); + + it('Throws if given uuid but missing metric.uuidField', () => { + function callCreateQuery() { + const options = { uuid: 'abc123', clusterUuid: 'cuid123', metric }; + delete options.metric.uuidField; + return createQuery(options); + } + expect(callCreateQuery).toThrowError(MissingRequiredError); + }); + + it('Uses `type` option to add type filter with minimal fields', () => { + const options = { type: 'cluster_stats', clusterUuid: 'cuid123', metric }; + expect(createQuery(options)).toEqual({ + bool: { + filter: [ + { bool: { should: [{ term: { type: 'cluster_stats' } }] } }, + { term: { cluster_uuid: 'cuid123' } }, + ], + }, + }); + }); + + it('Uses `type` option to add type filter with all other option fields and no data stream fields', () => { + const options = { + type: 'cluster_stats', + clusterUuid: 'cuid123', + uuid: 'abc123', + start: 1456826400000, + end: 14568264000000, + metric, + }; + expect(createQuery(options)).toEqual({ + bool: { + filter: [ + { bool: { should: [{ term: { type: 'cluster_stats' } }] } }, + { term: { cluster_uuid: 'cuid123' } }, + { term: { 'source_node.uuid': 'abc123' } }, + { + range: { + timestamp: { format: 'epoch_millis', gte: 1456826400000, lte: 14568264000000 }, + }, + }, + ], + }, + }); + }); + + it('Uses `dsType` option to add filter with all other option fields', () => { + const options = { + dsDataset: 'elasticsearch.cluster_stats', + clusterUuid: 'cuid123', + uuid: 'abc123', + start: 1456826400000, + end: 14568264000000, + metric, + }; + expect(createQuery(options)).toEqual({ + bool: { + filter: [ + { + bool: { + should: [{ term: { 'data_stream.dataset': 'elasticsearch.cluster_stats' } }], + }, + }, + { term: { cluster_uuid: 'cuid123' } }, + { term: { 'source_node.uuid': 'abc123' } }, + { + range: { + timestamp: { format: 'epoch_millis', gte: 1456826400000, lte: 14568264000000 }, + }, + }, + ], + }, + }); + }); + + it('Uses legacy `type`, `dsDataset`, `metricset` options to add type filters and data stream filters with minimal fields that defaults to `metrics` data_stream', () => { + const options = { + type: 'cluster_stats', + metricset: 'cluster_stats', + dsDataset: 'elasticsearch.cluster_stats', + clusterUuid: 'cuid123', + metric, + }; + expect(createQuery(options)).toEqual({ + bool: { + filter: [ + { + bool: { + should: [ + { + term: { + 'data_stream.dataset': 'elasticsearch.cluster_stats', + }, + }, + { + term: { + 'metricset.name': 'cluster_stats', + }, + }, + { term: { type: 'cluster_stats' } }, + ], + }, + }, + { term: { cluster_uuid: 'cuid123' } }, + ], + }, + }); + }); + + it('Uses legacy `type`, `metricset`, `dsDataset`, and `filters` options', () => { + const options = { + type: 'cluster_stats', + metricset: 'cluster_stats', + dsDataset: 'elasticsearch.cluster_stats', + clusterUuid: 'cuid123', + metric, + filters: [ + { + term: { 'source_node.uuid': `nuid123` }, + }, + ], + }; + expect(createQuery(options)).toEqual({ + bool: { + filter: [ + { + bool: { + should: [ + { term: { 'data_stream.dataset': 'elasticsearch.cluster_stats' } }, + { term: { 'metricset.name': 'cluster_stats' } }, + { term: { type: 'cluster_stats' } }, + ], + }, + }, + { term: { cluster_uuid: 'cuid123' } }, + { term: { 'source_node.uuid': 'nuid123' } }, + ], + }, + }); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/create_query.ts b/x-pack/plugins/monitoring/server/lib/create_query.ts index 8dead521d24fb4..051b0ed6b4f9ce 100644 --- a/x-pack/plugins/monitoring/server/lib/create_query.ts +++ b/x-pack/plugins/monitoring/server/lib/create_query.ts @@ -56,7 +56,9 @@ export function createTimeFilter(options: { * document UUIDs, start time and end time, and injecting additional filters. * * Options object: - * @param {String} options.type - `type` field value of the documents + * @param {string} options.type - `type` field value of the documents in legay .monitoring indices + * @param {string} options.dsDataset - `data_stream.dataset` field values of the documents + * @param {string} options.metricset - `metricset.name` field values of the documents * @param {Array} options.filters - additional filters to add to the `bool` section of the query. Default: [] * @param {string} options.clusterUuid - a UUID of the cluster. Required. * @param {string} options.uuid - a UUID of the metric to filter for, or `null` if UUID should not be part of the query @@ -64,30 +66,44 @@ export function createTimeFilter(options: { * @param {Date} options.end - numeric timestamp (optional) * @param {Metric} options.metric - Metric instance or metric fields object @see ElasticsearchMetric.getMetricFields */ -export function createQuery(options: { + +interface CreateQueryOptions { type?: string; - types?: string[]; + dsDataset?: string; + metricset?: string; filters?: any[]; clusterUuid: string; uuid?: string; start?: number; end?: number; metric?: { uuidField?: string; timestampField: string }; -}) { - const { type, types, clusterUuid, uuid, filters } = defaults(options, { filters: [] }); +} +export function createQuery(options: CreateQueryOptions) { + const { type, metricset, dsDataset, clusterUuid, uuid, filters } = defaults(options, { + filters: [], + }); const isFromStandaloneCluster = clusterUuid === STANDALONE_CLUSTER_CLUSTER_UUID; + const terms = []; let typeFilter: any; + + // data_stream.dataset matches agent integration data streams + if (dsDataset) { + terms.push({ term: { 'data_stream.dataset': dsDataset } }); + } + // metricset.name matches standalone beats + if (metricset) { + terms.push({ term: { 'metricset.name': metricset } }); + } + // type matches legacy data if (type) { - typeFilter = { bool: { should: [{ term: { type } }, { term: { 'metricset.name': type } }] } }; - } else if (types) { + terms.push({ term: { type } }); + } + if (terms.length) { typeFilter = { bool: { - should: [ - ...types.map((t) => ({ term: { type: t } })), - ...types.map((t) => ({ term: { 'metricset.name': t } })), - ], + should: [...terms], }, }; } diff --git a/x-pack/plugins/monitoring/server/lib/details/get_metrics.test.js b/x-pack/plugins/monitoring/server/lib/details/get_metrics.test.js index 80234ee369aeef..5ba7fd1207448a 100644 --- a/x-pack/plugins/monitoring/server/lib/details/get_metrics.test.js +++ b/x-pack/plugins/monitoring/server/lib/details/get_metrics.test.js @@ -16,6 +16,18 @@ import aggMetricsBuckets from './__fixtures__/agg_metrics_buckets'; const min = 1498968000000; // 2017-07-02T04:00:00.000Z const max = 1499054399999; // 2017-07-03T03:59:59.999Z +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + function getMockReq(metricsBuckets = []) { const config = { get: sinon.stub(), @@ -59,27 +71,25 @@ function getMockReq(metricsBuckets = []) { }; } -const indexPattern = []; - describe('getMetrics and getSeries', () => { it('should return metrics with non-derivative metric', async () => { const req = getMockReq(nonDerivMetricsBuckets); const metricSet = ['node_cpu_utilization']; - const result = await getMetrics(req, indexPattern, metricSet); + const result = await getMetrics(req, 'elasticsearch', metricSet); expect(result).toMatchSnapshot(); }); it('should return metrics with derivative metric', async () => { const req = getMockReq(derivMetricsBuckets); const metricSet = ['cluster_search_request_rate']; - const result = await getMetrics(req, indexPattern, metricSet); + const result = await getMetrics(req, 'elasticsearch', metricSet); expect(result).toMatchSnapshot(); }); it('should return metrics with metric containing custom aggs', async () => { const req = getMockReq(aggMetricsBuckets); const metricSet = ['cluster_index_latency']; - const result = await getMetrics(req, indexPattern, metricSet); + const result = await getMetrics(req, 'elasticsearch', metricSet); expect(result).toMatchSnapshot(); }); @@ -91,14 +101,14 @@ describe('getMetrics and getSeries', () => { keys: ['index_mem_fixed_bit_set', 'index_mem_versions'], }, ]; - const result = await getMetrics(req, indexPattern, metricSet); + const result = await getMetrics(req, 'elasticsearch', metricSet); expect(result).toMatchSnapshot(); }); it('should return metrics with metric that uses default calculation', async () => { const req = getMockReq(nonDerivMetricsBuckets); const metricSet = ['kibana_max_response_times']; - const result = await getMetrics(req, indexPattern, metricSet); + const result = await getMetrics(req, 'elasticsearch', metricSet); expect(result).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/monitoring/server/lib/details/get_metrics.ts b/x-pack/plugins/monitoring/server/lib/details/get_metrics.ts index a8a1117839dfe8..efca0b84b0d65c 100644 --- a/x-pack/plugins/monitoring/server/lib/details/get_metrics.ts +++ b/x-pack/plugins/monitoring/server/lib/details/get_metrics.ts @@ -11,20 +11,21 @@ import { getSeries } from './get_series'; import { calculateTimeseriesInterval } from '../calculate_timeseries_interval'; import { getTimezone } from '../get_timezone'; import { LegacyRequest } from '../../types'; +import { INDEX_PATTERN_TYPES } from '../../../common/constants'; type Metric = string | { keys: string | string[]; name: string }; // TODO: Switch to an options object argument here export async function getMetrics( req: LegacyRequest, - indexPattern: string, + moduleType: INDEX_PATTERN_TYPES, metricSet: Metric[] = [], filters: Array> = [], metricOptions: Record = {}, numOfBuckets: number = 0, groupBy: string | Record | null = null ) { - checkParam(indexPattern, 'indexPattern in details/getMetrics'); + checkParam(moduleType, 'moduleType in details/getMetrics'); checkParam(metricSet, 'metricSet in details/getMetrics'); const config = req.server.config(); @@ -53,7 +54,7 @@ export async function getMetrics( return Promise.all( metricNames.map((metricName) => { - return getSeries(req, indexPattern, metricName, metricOptions, filters, groupBy, { + return getSeries(req, moduleType, metricName, metricOptions, filters, groupBy, { min, max, bucketSize, diff --git a/x-pack/plugins/monitoring/server/lib/details/get_series.ts b/x-pack/plugins/monitoring/server/lib/details/get_series.ts index 99652cbff3ffd8..3a053b16aad7cd 100644 --- a/x-pack/plugins/monitoring/server/lib/details/get_series.ts +++ b/x-pack/plugins/monitoring/server/lib/details/get_series.ts @@ -16,9 +16,12 @@ import { formatTimestampToDuration } from '../../../common'; import { NORMALIZED_DERIVATIVE_UNIT, CALCULATE_DURATION_UNTIL, + INDEX_PATTERN_TYPES, STANDALONE_CLUSTER_CLUSTER_UUID, } from '../../../common/constants'; import { formatUTCTimestampForTimezone } from '../format_timezone'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; type SeriesBucket = Bucket & { metric_mb_deriv?: { normalized_value: number } }; @@ -117,7 +120,7 @@ function createMetricAggs(metric: Metric) { async function fetchSeries( req: LegacyRequest, - indexPattern: string, + moduleType: INDEX_PATTERN_TYPES, metric: Metric, metricOptions: any, groupBy: string | Record | null, @@ -175,8 +178,14 @@ async function fetchSeries( }; } + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + ccs: req.payload.ccs, + }); + const params = { - index: indexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, body: { @@ -327,14 +336,15 @@ function handleSeries( * TODO: This should be expanded to accept multiple metrics in a single request to allow a single date histogram to be used. * * @param {Object} req The incoming user's request. - * @param {String} indexPattern The relevant index pattern (not just for Elasticsearch!). + * @param {String} moduleType The relevant module eg: elasticsearch, kibana, logstash. * @param {String} metricName The name of the metric being plotted. * @param {Array} filters Any filters that should be applied to the query. * @return {Promise} The object response containing the {@code timeRange}, {@code metric}, and {@code data}. */ + export async function getSeries( req: LegacyRequest, - indexPattern: string, + moduleType: INDEX_PATTERN_TYPES, metricName: string, metricOptions: Record, filters: Array>, @@ -346,7 +356,7 @@ export async function getSeries( timezone, }: { min: string | number; max: string | number; bucketSize: number; timezone: string } ) { - checkParam(indexPattern, 'indexPattern in details/getSeries'); + checkParam(moduleType, 'moduleType in details/getSeries'); const metric = metrics[metricName]; if (!metric) { @@ -354,7 +364,7 @@ export async function getSeries( } const response = await fetchSeries( req, - indexPattern, + moduleType, metric, metricOptions, groupBy, diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/ccr.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/ccr.ts index 17dc48d0b237e7..902e4bf784a096 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/ccr.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/ccr.ts @@ -14,9 +14,18 @@ import { ElasticsearchMetric } from '../metrics'; import { createQuery } from '../create_query'; import { ElasticsearchResponse } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; -export async function checkCcrEnabled(req: LegacyRequest, esIndexPattern: string) { - checkParam(esIndexPattern, 'esIndexPattern in checkCcrEnabled'); +export async function checkCcrEnabled(req: LegacyRequest, ccs: string) { + const dataset = 'cluster_stats'; + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + ccs, + }); const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); @@ -25,12 +34,14 @@ export async function checkCcrEnabled(req: LegacyRequest, esIndexPattern: string const metricFields = ElasticsearchMetric.getMetricFields(); const params = { - index: esIndexPattern, + index: indexPatterns, size: 1, ignore_unavailable: true, body: { query: createQuery({ - type: 'cluster_stats', + type: dataset, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts index 52c5b5b97f77b4..7f6eea717be763 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts @@ -19,6 +19,8 @@ import { ElasticsearchResponseHit, } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; /** * Filter out shard activity that we do not care about. @@ -86,37 +88,63 @@ export function handleMbLastRecoveries(resp: ElasticsearchResponse, start: numbe return filtered; } -export async function getLastRecovery( - req: LegacyRequest, - esIndexPattern: string, - esIndexPatternEcs: string, - size: number -) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getLastRecovery'); - +export async function getLastRecovery(req: LegacyRequest, size: number) { const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; const clusterUuid = req.params.clusterUuid; const metric = ElasticsearchMetric.getMetricFields(); + + const dataset = 'index_recovery'; + const moduleType = 'elasticsearch'; + const indexPattern = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + ccs: req.payload.ccs, + }); + const legacyParams = { - index: esIndexPattern, + index: indexPattern, size: 1, ignore_unavailable: true, body: { _source: ['index_recovery.shards'], sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, - query: createQuery({ type: 'index_recovery', start, end, clusterUuid, metric }), + query: createQuery({ + type: dataset, + metricset: dataset, + start, + end, + clusterUuid, + metric, + }), }, }; + + const indexPatternEcs = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + ccs: req.payload.ccs, + ecsLegacyOnly: true, + }); const ecsParams = { - index: esIndexPatternEcs, + index: indexPatternEcs, size, ignore_unavailable: true, body: { _source: ['elasticsearch.index.recovery', '@timestamp'], sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, - query: createQuery({ type: 'index_recovery', start, end, clusterUuid, metric }), + query: createQuery({ + type: dataset, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, + start, + end, + clusterUuid, + metric, + }), aggs: { max_timestamp: { max: { diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.ts index b0c82bdc0f5023..f321326ee090a5 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.ts @@ -15,6 +15,8 @@ import { ElasticsearchMetric } from '../metrics'; import { ML_SUPPORTED_LICENSES } from '../../../common/constants'; import { ElasticsearchResponse, ElasticsearchSource } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; /* * Get a listing of jobs along with some metric data to use for the listing @@ -37,17 +39,26 @@ export function handleResponse(response: ElasticsearchResponse) { export type MLJobs = ReturnType; -export function getMlJobs(req: LegacyRequest, esIndexPattern: string) { - checkParam(esIndexPattern, 'esIndexPattern in getMlJobs'); - +export function getMlJobs(req: LegacyRequest) { const config = req.server.config(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const start = req.payload.timeRange.min; // no wrapping in moment :) const end = req.payload.timeRange.max; const clusterUuid = req.params.clusterUuid; const metric = ElasticsearchMetric.getMetricFields(); + + const dataset = 'ml_job'; + const type = 'job_stats'; + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); + const params = { - index: esIndexPattern, + index: indexPatterns, size: maxBucketSize, ignore_unavailable: true, filter_path: [ @@ -69,7 +80,15 @@ export function getMlJobs(req: LegacyRequest, esIndexPattern: string) { body: { sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, collapse: { field: 'job_stats.job_id' }, - query: createQuery({ types: ['ml_job', 'job_stats'], start, end, clusterUuid, metric }), + query: createQuery({ + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, + start, + end, + clusterUuid, + metric, + }), }, }; @@ -81,11 +100,7 @@ export function getMlJobs(req: LegacyRequest, esIndexPattern: string) { * cardinality isn't guaranteed to be accurate is the issue * but it will be as long as the precision threshold is >= the actual value */ -export function getMlJobsForCluster( - req: LegacyRequest, - esIndexPattern: string, - cluster: ElasticsearchSource -) { +export function getMlJobsForCluster(req: LegacyRequest, cluster: ElasticsearchSource, ccs: string) { const license = cluster.license ?? cluster.elasticsearch?.cluster?.stats?.license ?? {}; if (license.status === 'active' && includes(ML_SUPPORTED_LICENSES, license.type)) { @@ -94,19 +109,37 @@ export function getMlJobsForCluster( const end = req.payload.timeRange.max; const clusterUuid = req.params.clusterUuid; const metric = ElasticsearchMetric.getMetricFields(); + + const type = 'job_stats'; + const dataset = 'ml_job'; + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + ccs, + }); + const params = { - index: esIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, filter_path: 'aggregations.jobs_count.value', body: { - query: createQuery({ types: ['ml_job', 'job_stats'], start, end, clusterUuid, metric }), + query: createQuery({ + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, + start, + end, + clusterUuid, + metric, + }), aggs: { jobs_count: { cardinality: { field: 'job_stats.job_id' } }, }, }, }; - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); return callWithRequest(req, 'search', params).then((response: ElasticsearchResponse) => { diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_index_summary.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_index_summary.ts index 8a03027a93a565..388c3a364c77af 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_index_summary.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_index_summary.ts @@ -15,6 +15,8 @@ import { createQuery } from '../../create_query'; import { ElasticsearchMetric } from '../../metrics'; import { ElasticsearchResponse } from '../../../../common/types/es'; import { LegacyRequest } from '../../../types'; +import { getNewIndexPatterns } from '../../cluster/get_index_patterns'; +import { Globals } from '../../../static_globals'; export function handleResponse(shardStats: any, indexUuid: string) { return (response: ElasticsearchResponse) => { @@ -64,7 +66,6 @@ export function handleResponse(shardStats: any, indexUuid: string) { export function getIndexSummary( req: LegacyRequest, - esIndexPattern: string, shardStats: any, { clusterUuid, @@ -73,7 +74,15 @@ export function getIndexSummary( end, }: { clusterUuid: string; indexUuid: string; start: number; end: number } ) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getIndexSummary'); + const dataset = 'index'; // data_stream.dataset + const type = 'index_stats'; // legacy + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + dataset, + moduleType, + ccs: req.payload.ccs, + }); const metric = ElasticsearchMetric.getMetricFields(); const filters = [ @@ -87,13 +96,15 @@ export function getIndexSummary( }, ]; const params = { - index: esIndexPattern, + index: indexPatterns, size: 1, ignore_unavailable: true, body: { sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, query: createQuery({ - types: ['index', 'index_stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.ts index a43feb8fc84a3b..85e49f463526a6 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.ts @@ -19,6 +19,8 @@ import { calculateRate } from '../../calculate_rate'; import { getUnassignedShards } from '../shards'; import { ElasticsearchResponse } from '../../../../common/types/es'; import { LegacyRequest } from '../../../types'; +import { getNewIndexPatterns } from '../../cluster/get_index_patterns'; +import { Globals } from '../../../static_globals'; export function handleResponse( resp: ElasticsearchResponse, @@ -95,7 +97,7 @@ export function handleResponse( } export function buildGetIndicesQuery( - esIndexPattern: string, + req: LegacyRequest, clusterUuid: string, { start, @@ -113,9 +115,18 @@ export function buildGetIndicesQuery( }); } const metricFields = ElasticsearchMetric.getMetricFields(); + const dataset = 'index'; // data_stream.dataset + const type = 'index_stats'; // legacy + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + dataset, + moduleType, + }); return { - index: esIndexPattern, + index: indexPatterns, size, ignore_unavailable: true, filter_path: [ @@ -145,7 +156,9 @@ export function buildGetIndicesQuery( ], body: { query: createQuery({ - types: ['index', 'index_stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, @@ -167,17 +180,14 @@ export function buildGetIndicesQuery( export function getIndices( req: LegacyRequest, - esIndexPattern: string, showSystemIndices: boolean = false, shardStats: any ) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getIndices'); - const { min: start, max: end } = req.payload.timeRange; const clusterUuid = req.params.clusterUuid; const config = req.server.config(); - const params = buildGetIndicesQuery(esIndexPattern, clusterUuid, { + const params = buildGetIndicesQuery(req, clusterUuid, { start, end, showSystemIndices, diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_node_summary.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_node_summary.ts index aed6b40675e459..a422ccf95527c4 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_node_summary.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_node_summary.ts @@ -24,6 +24,8 @@ import { ElasticsearchLegacySource, } from '../../../../common/types/es'; import { LegacyRequest } from '../../../types'; +import { getNewIndexPatterns } from '../../cluster/get_index_patterns'; +import { Globals } from '../../../static_globals'; export function handleResponse( clusterState: ElasticsearchSource['cluster_state'], @@ -100,7 +102,6 @@ export function handleResponse( export function getNodeSummary( req: LegacyRequest, - esIndexPattern: string, clusterState: ElasticsearchSource['cluster_state'], shardStats: any, { @@ -110,25 +111,40 @@ export function getNodeSummary( end, }: { clusterUuid: string; nodeUuid: string; start: number; end: number } ) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getNodeSummary'); - - // Build up the Elasticsearch request const metric = ElasticsearchMetric.getMetricFields(); const filters = [ { term: { 'source_node.uuid': nodeUuid }, }, ]; + + const dataset = 'node_stats'; + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + dataset, + moduleType, + }); + const params = { - index: esIndexPattern, + index: indexPatterns, size: 1, ignore_unavailable: true, body: { sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, - query: createQuery({ type: 'node_stats', start, end, clusterUuid, metric, filters }), + query: createQuery({ + type: dataset, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, + start, + end, + clusterUuid, + metric, + filters, + }), }, }; - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); return callWithRequest(req, 'search', params).then( handleResponse(clusterState, shardStats, nodeUuid) diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.test.js b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.test.js index e6f04e3c649e54..738ea9ecc6a367 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.test.js +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.test.js @@ -7,6 +7,18 @@ import { getNodeIds } from './get_node_ids'; +jest.mock('../../../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('getNodeIds', () => { it('should return a list of ids and uuids', async () => { const callWithRequest = jest.fn().mockReturnValue({ @@ -37,6 +49,9 @@ describe('getNodeIds', () => { }, }, server: { + config: () => ({ + get: () => true, + }), plugins: { elasticsearch: { getCluster: () => ({ diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.ts index 03524eebd12e8c..58f4a0b8aca56c 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_node_ids.ts @@ -10,16 +10,26 @@ import { get } from 'lodash'; import { ElasticsearchMetric } from '../../../metrics'; import { createQuery } from '../../../create_query'; import { LegacyRequest, Bucket } from '../../../../types'; +import { getNewIndexPatterns } from '../../../cluster/get_index_patterns'; +import { Globals } from '../../../../static_globals'; export async function getNodeIds( req: LegacyRequest, - indexPattern: string, { clusterUuid }: { clusterUuid: string }, size: number ) { const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); + const dataset = 'node_stats'; + const moduleType = 'elasticsearch'; + const indexPattern = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); + const params = { index: indexPattern, size: 0, @@ -27,7 +37,9 @@ export async function getNodeIds( filter_path: ['aggregations.composite_data.buckets'], body: { query: createQuery({ - type: 'node_stats', + type: dataset, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, metric: ElasticsearchMetric.getMetricFields(), diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.ts index 0b5c0337e6c475..2bd7078fa00a4f 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.ts @@ -6,7 +6,6 @@ */ import moment from 'moment'; -import { checkParam } from '../../../error_missing_required'; import { createQuery } from '../../../create_query'; import { calculateAuto } from '../../../calculate_auto'; import { ElasticsearchMetric } from '../../../metrics'; @@ -15,6 +14,8 @@ import { handleResponse } from './handle_response'; import { LISTING_METRICS_NAMES, LISTING_METRICS_PATHS } from './nodes_listing_metrics'; import { LegacyRequest } from '../../../../types'; import { ElasticsearchModifiedSource } from '../../../../../common/types/es'; +import { getNewIndexPatterns } from '../../../cluster/get_index_patterns'; +import { Globals } from '../../../../static_globals'; /* Run an aggregation on node_stats to get stat data for the selected time * range for all the active nodes. Every option is a key to a configuration @@ -35,13 +36,10 @@ import { ElasticsearchModifiedSource } from '../../../../../common/types/es'; */ export async function getNodes( req: LegacyRequest, - esIndexPattern: string, pageOfNodes: Array<{ uuid: string }>, clusterStats: ElasticsearchModifiedSource, nodesShardCount: { nodes: { [nodeId: string]: { shardCount: number } } } ) { - checkParam(esIndexPattern, 'esIndexPattern in getNodes'); - const start = moment.utc(req.payload.timeRange.min).valueOf(); const orgStart = start; const end = moment.utc(req.payload.timeRange.max).valueOf(); @@ -67,13 +65,24 @@ export async function getNodes( }, ]; + const dataset = 'node_stats'; + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); + const params = { - index: esIndexPattern, + index: indexPatterns, size: config.get('monitoring.ui.max_bucket_size'), ignore_unavailable: true, body: { query: createQuery({ - type: 'node_stats', + type: dataset, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, @@ -112,7 +121,6 @@ export async function getNodes( ...LISTING_METRICS_PATHS, ], }; - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); const response = await callWithRequest(req, 'search', params); diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.test.js b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.test.js index c7939027a0fb32..f96cb8e7a18539 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.test.js +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.test.js @@ -47,7 +47,6 @@ describe('getPaginatedNodes', () => { }), }, }; - const esIndexPattern = '.monitoring-es-*'; const clusterUuid = '1abc'; const metricSet = ['foo', 'bar']; const pagination = { index: 0, size: 10 }; @@ -74,7 +73,6 @@ describe('getPaginatedNodes', () => { it('should return a subset based on the pagination parameters', async () => { const nodes = await getPaginatedNodes( req, - esIndexPattern, { clusterUuid }, metricSet, pagination, @@ -94,7 +92,6 @@ describe('getPaginatedNodes', () => { it('should return a sorted subset', async () => { const nodes = await getPaginatedNodes( req, - esIndexPattern, { clusterUuid }, metricSet, pagination, @@ -111,17 +108,11 @@ describe('getPaginatedNodes', () => { }); }); - it('should return a filterd subset', async () => { - const nodes = await getPaginatedNodes( - req, - esIndexPattern, - { clusterUuid }, - metricSet, - pagination, - sort, - 'tw', - { clusterStats, nodesShardCount } - ); + it('should return a filtered subset', async () => { + const nodes = await getPaginatedNodes(req, { clusterUuid }, metricSet, pagination, sort, 'tw', { + clusterStats, + nodesShardCount, + }); expect(nodes).toEqual({ pageOfNodes: [{ name: 'two', uuid: 2, isOnline: false, shardCount: 5, foo: 12 }], totalNodeCount: 1, diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.ts index 118140fe3f9cd2..d353ea844cdf92 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.ts @@ -41,7 +41,6 @@ interface Node { export async function getPaginatedNodes( req: LegacyRequest, - esIndexPattern: string, { clusterUuid }: { clusterUuid: string }, metricSet: string[], pagination: { index: number; size: number }, @@ -59,7 +58,7 @@ export async function getPaginatedNodes( ) { const config = req.server.config(); const size = Number(config.get('monitoring.ui.max_bucket_size')); - const nodes: Node[] = await getNodeIds(req, esIndexPattern, { clusterUuid }, size); + const nodes: Node[] = await getNodeIds(req, { clusterUuid }, size); // Add `isOnline` and shards from the cluster state and shard stats const clusterState = clusterStats?.cluster_state ?? { nodes: {} }; @@ -87,13 +86,14 @@ export async function getPaginatedNodes( }; const metricSeriesData = await getMetrics( req, - esIndexPattern, + 'elasticsearch', metricSet, filters, { nodes }, 4, groupBy ); + for (const metricName in metricSeriesData) { if (!metricSeriesData.hasOwnProperty(metricName)) { continue; diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.test.js b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.test.js index 6521f1f435cbc7..6a9e01166ea875 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.test.js +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.test.js @@ -7,6 +7,18 @@ import { getIndicesUnassignedShardStats } from './get_indices_unassigned_shard_stats'; +jest.mock('../../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('getIndicesUnassignedShardStats', () => { it('should return the unassigned shard stats for indices', async () => { const indices = { @@ -16,6 +28,7 @@ describe('getIndicesUnassignedShardStats', () => { }; const req = { + payload: {}, server: { config: () => ({ get: () => {}, @@ -52,9 +65,8 @@ describe('getIndicesUnassignedShardStats', () => { }, }, }; - const esIndexPattern = '*'; const cluster = {}; - const stats = await getIndicesUnassignedShardStats(req, esIndexPattern, cluster); + const stats = await getIndicesUnassignedShardStats(req, cluster); expect(stats.indices).toEqual(indices); }); }); diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.ts index 87f79ff5b9b44d..b0ba916b53eb8e 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.ts @@ -16,12 +16,10 @@ import { ElasticsearchMetric } from '../../metrics'; import { calculateIndicesTotals } from './calculate_shard_stat_indices_totals'; import { LegacyRequest } from '../../../types'; import { ElasticsearchModifiedSource } from '../../../../common/types/es'; +import { getNewIndexPatterns } from '../../cluster/get_index_patterns'; +import { Globals } from '../../../static_globals'; -async function getUnassignedShardData( - req: LegacyRequest, - esIndexPattern: string, - cluster: ElasticsearchModifiedSource -) { +async function getUnassignedShardData(req: LegacyRequest, cluster: ElasticsearchModifiedSource) { const config = req.server.config(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const metric = ElasticsearchMetric.getMetricFields(); @@ -38,14 +36,26 @@ async function getUnassignedShardData( }); } + const dataset = 'shard'; // data_stream.dataset + const type = 'shards'; // legacy + const moduleType = 'elasticsearch'; + const indexPattern = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); + const params = { - index: esIndexPattern, + index: indexPattern, size: 0, ignore_unavailable: true, body: { sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, query: createQuery({ - types: ['shard', 'shards'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, clusterUuid: cluster.cluster_uuid ?? cluster.elasticsearch?.cluster?.id, metric, filters, @@ -84,12 +94,9 @@ async function getUnassignedShardData( export async function getIndicesUnassignedShardStats( req: LegacyRequest, - esIndexPattern: string, cluster: ElasticsearchModifiedSource ) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardStats'); - - const response = await getUnassignedShardData(req, esIndexPattern, cluster); + const response = await getUnassignedShardData(req, cluster); const indices = get(response, 'aggregations.indices.buckets', []).reduce( (accum: any, bucket: any) => { const index = bucket.key; diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.test.js b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.test.js index 16d8693ca931dd..c5c77ef389427c 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.test.js +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.test.js @@ -7,6 +7,18 @@ import { getNodesShardCount } from './get_nodes_shard_count'; +jest.mock('../../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + describe('getNodeShardCount', () => { it('should return the shard count per node', async () => { const nodes = { @@ -16,6 +28,7 @@ describe('getNodeShardCount', () => { }; const req = { + payload: {}, server: { config: () => ({ get: () => {}, @@ -38,9 +51,8 @@ describe('getNodeShardCount', () => { }, }, }; - const esIndexPattern = '*'; const cluster = {}; - const counts = await getNodesShardCount(req, esIndexPattern, cluster); + const counts = await getNodesShardCount(req, cluster); expect(counts.nodes).toEqual(nodes); }); }); diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.ts index 12ce144ebf5c5a..a8ce0d429f06c2 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.ts @@ -14,12 +14,10 @@ import { createQuery } from '../../create_query'; import { ElasticsearchMetric } from '../../metrics'; import { LegacyRequest } from '../../../types'; import { ElasticsearchModifiedSource } from '../../../../common/types/es'; +import { getNewIndexPatterns } from '../../cluster/get_index_patterns'; +import { Globals } from '../../../static_globals'; -async function getShardCountPerNode( - req: LegacyRequest, - esIndexPattern: string, - cluster: ElasticsearchModifiedSource -) { +async function getShardCountPerNode(req: LegacyRequest, cluster: ElasticsearchModifiedSource) { const config = req.server.config(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const metric = ElasticsearchMetric.getMetricFields(); @@ -35,15 +33,26 @@ async function getShardCountPerNode( }, }); } + const dataset = 'shard'; // data_stream.dataset + const type = 'shards'; // legacy + const moduleType = 'elasticsearch'; + const indexPattern = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const params = { - index: esIndexPattern, + index: indexPattern, size: 0, ignore_unavailable: true, body: { sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, query: createQuery({ - types: ['shard', 'shards'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, clusterUuid: cluster.cluster_uuid ?? cluster.elasticsearch?.cluster?.id, metric, filters, @@ -63,14 +72,8 @@ async function getShardCountPerNode( return await callWithRequest(req, 'search', params); } -export async function getNodesShardCount( - req: LegacyRequest, - esIndexPattern: string, - cluster: ElasticsearchModifiedSource -) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardStats'); - - const response = await getShardCountPerNode(req, esIndexPattern, cluster); +export async function getNodesShardCount(req: LegacyRequest, cluster: ElasticsearchModifiedSource) { + const response = await getShardCountPerNode(req, cluster); const nodes = get(response, 'aggregations.nodes.buckets', []).reduce( (accum: any, bucket: any) => { accum[bucket.key] = { shardCount: bucket.doc_count }; diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.ts index 26a1f88c719cf6..cede4bf921419b 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.ts @@ -6,13 +6,16 @@ */ // @ts-ignore -import { checkParam } from '../../error_missing_required'; +import { StringOptions } from '@kbn/config-schema/target_types/types'; // @ts-ignore import { createQuery } from '../../create_query'; // @ts-ignore import { ElasticsearchMetric } from '../../metrics'; import { ElasticsearchResponse, ElasticsearchLegacySource } from '../../../../common/types/es'; import { LegacyRequest } from '../../../types'; +import { getNewIndexPatterns } from '../../cluster/get_index_patterns'; +import { Globals } from '../../../static_globals'; + export function handleResponse(response: ElasticsearchResponse) { const hits = response.hits?.hits; if (!hits) { @@ -57,15 +60,12 @@ export function handleResponse(response: ElasticsearchResponse) { export function getShardAllocation( req: LegacyRequest, - esIndexPattern: string, { shardFilter, stateUuid, showSystemIndices = false, }: { shardFilter: any; stateUuid: string; showSystemIndices: boolean } ) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardAllocation'); - const filters = [ { bool: { @@ -100,15 +100,32 @@ export function getShardAllocation( const config = req.server.config(); const clusterUuid = req.params.clusterUuid; const metric = ElasticsearchMetric.getMetricFields(); + + const dataset = 'shard'; // data_stream.dataset + const type = 'shards'; // legacy + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + dataset, + moduleType, + }); + const params = { - index: esIndexPattern, + index: indexPatterns, size: config.get('monitoring.ui.max_bucket_size'), ignore_unavailable: true, body: { - query: createQuery({ types: ['shard', 'shards'], clusterUuid, metric, filters }), + query: createQuery({ + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, + clusterUuid, + metric, + filters, + }), }, }; - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); return callWithRequest(req, 'search', params).then(handleResponse); } diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stats.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stats.ts index 0ee047b2b938b3..29cbcb9ac0567b 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stats.ts @@ -20,6 +20,8 @@ import { getShardAggs } from './get_shard_stat_aggs'; import { calculateIndicesTotals } from './calculate_shard_stat_indices_totals'; import { LegacyRequest } from '../../../types'; import { ElasticsearchResponse, ElasticsearchModifiedSource } from '../../../../common/types/es'; +import { getNewIndexPatterns } from '../../cluster/get_index_patterns'; +import { Globals } from '../../../static_globals'; export function handleResponse( resp: ElasticsearchResponse, @@ -55,11 +57,18 @@ export function handleResponse( export function getShardStats( req: LegacyRequest, - esIndexPattern: string, cluster: ElasticsearchModifiedSource, { includeNodes = false, includeIndices = false, indexName = null, nodeUuid = null } = {} ) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardStats'); + const dataset = 'shard'; // data_stream.dataset + const type = 'shards'; // legacy + const moduleType = 'elasticsearch'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const config = req.server.config(); const metric = ElasticsearchMetric.getMetricFields(); @@ -95,13 +104,15 @@ export function getShardStats( }); } const params = { - index: esIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, body: { sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, query: createQuery({ - types: ['shard', 'shards'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, clusterUuid: cluster.cluster_uuid ?? cluster.elasticsearch?.cluster?.id, metric, filters, @@ -111,7 +122,6 @@ export function getShardStats( }, }, }; - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); return callWithRequest(req, 'search', params).then((resp) => { return handleResponse(resp, includeNodes, includeIndices, cluster); diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/normalize_shard_objects.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/normalize_shard_objects.ts index 6aa84a0809e963..c6f38c7ac795f8 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/normalize_shard_objects.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/shards/normalize_shard_objects.ts @@ -48,7 +48,8 @@ export function normalizeNodeShards(masterNode: string) { [node.key]: { shardCount: node.doc_count, indexCount: node.index_count.value, - name: node.node_names.buckets[0].key, + // this field is always missing, problem with package? elasticsearch.node.name ECS field doesn't exist + name: node.node_names.buckets[0]?.key || 'NO NAME', node_ids: nodeIds, type: calculateNodeType(_node, masterNode), // put the "star" icon on the node link in the shard allocator }, diff --git a/x-pack/plugins/monitoring/server/lib/enterprise_search/create_enterprise_search_query.ts b/x-pack/plugins/monitoring/server/lib/enterprise_search/create_enterprise_search_query.ts index 3a418e010e1f74..5e5c972d2ba460 100644 --- a/x-pack/plugins/monitoring/server/lib/enterprise_search/create_enterprise_search_query.ts +++ b/x-pack/plugins/monitoring/server/lib/enterprise_search/create_enterprise_search_query.ts @@ -16,7 +16,6 @@ import { STANDALONE_CLUSTER_CLUSTER_UUID } from '../../../common/constants'; */ export function createEnterpriseSearchQuery(options: { filters?: any[]; - types?: string[]; metric?: EnterpriseSearchMetricFields; uuid?: string; start?: number; @@ -25,7 +24,6 @@ export function createEnterpriseSearchQuery(options: { const opts = { filters: [] as any[], metric: EnterpriseSearchMetric.getMetricFields(), - types: ['health', 'stats'], clusterUuid: STANDALONE_CLUSTER_CLUSTER_UUID, // This is to disable the stack monitoring clusterUuid filter ...(options ?? {}), }; @@ -33,6 +31,26 @@ export function createEnterpriseSearchQuery(options: { opts.filters.push({ bool: { should: [ + { + term: { + type: 'health', + }, + }, + { + term: { + type: 'stats', + }, + }, + { + term: { + 'metricset.name': 'health', + }, + }, + { + term: { + 'metricset.name': 'stats', + }, + }, { term: { 'event.dataset': 'enterprisesearch.health' } }, { term: { 'event.dataset': 'enterprisesearch.stats' } }, ], diff --git a/x-pack/plugins/monitoring/server/lib/enterprise_search/get_enterprise_search_for_clusters.ts b/x-pack/plugins/monitoring/server/lib/enterprise_search/get_enterprise_search_for_clusters.ts index 96ba1d18dc9e80..d46853fe48d3f1 100644 --- a/x-pack/plugins/monitoring/server/lib/enterprise_search/get_enterprise_search_for_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/enterprise_search/get_enterprise_search_for_clusters.ts @@ -7,7 +7,6 @@ import { ElasticsearchResponse } from '../../../common/types/es'; import { LegacyRequest, Cluster } from '../../types'; -import { checkParam } from '../error_missing_required'; import { createEnterpriseSearchQuery } from './create_enterprise_search_query'; import { EnterpriseSearchMetric } from '../metrics'; import { @@ -15,6 +14,8 @@ import { entSearchAggResponseHandler, entSearchUuidsAgg, } from './_enterprise_search_stats'; +import { getLegacyIndexPattern } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; function handleResponse(clusterUuid: string, response: ElasticsearchResponse) { const stats = entSearchAggResponseHandler(response); @@ -27,24 +28,25 @@ function handleResponse(clusterUuid: string, response: ElasticsearchResponse) { export function getEnterpriseSearchForClusters( req: LegacyRequest, - entSearchIndexPattern: string, - clusters: Cluster[] + clusters: Cluster[], + ccs: string ) { - checkParam( - entSearchIndexPattern, - 'entSearchIndexPattern in enterprise_earch/getEnterpriseSearchForClusters' - ); - const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; const config = req.server.config(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); + const indexPatterns = getLegacyIndexPattern({ + moduleType: 'enterprisesearch', + ccs, + config: Globals.app.config, + }); + return Promise.all( clusters.map(async (cluster) => { const clusterUuid = cluster.elasticsearch?.cluster?.id ?? cluster.cluster_uuid; const params = { - index: entSearchIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, filter_path: entSearchAggFilterPath, diff --git a/x-pack/plugins/monitoring/server/lib/enterprise_search/get_stats.ts b/x-pack/plugins/monitoring/server/lib/enterprise_search/get_stats.ts index bcb5617e0c9d87..73bd33daeac34d 100644 --- a/x-pack/plugins/monitoring/server/lib/enterprise_search/get_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/enterprise_search/get_stats.ts @@ -8,28 +8,30 @@ import moment from 'moment'; import { ElasticsearchResponse } from '../../../common/types/es'; import { LegacyRequest } from '../../types'; -import { checkParam } from '../error_missing_required'; import { createEnterpriseSearchQuery } from './create_enterprise_search_query'; import { entSearchAggFilterPath, entSearchUuidsAgg, entSearchAggResponseHandler, } from './_enterprise_search_stats'; +import { getLegacyIndexPattern } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; -export async function getStats( - req: LegacyRequest, - entSearchIndexPattern: string, - clusterUuid: string -) { - checkParam(entSearchIndexPattern, 'entSearchIndexPattern in getStats'); - +export async function getStats(req: LegacyRequest, clusterUuid: string) { const config = req.server.config(); const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); + // just get the legacy pattern since no integration exists yet + const indexPattern = getLegacyIndexPattern({ + moduleType: 'enterprisesearch', + config: Globals.app.config, + ccs: req.payload.ccs, + }); + const params = { - index: entSearchIndexPattern, + index: indexPattern, filter_path: entSearchAggFilterPath, size: 0, ignore_unavailable: true, diff --git a/x-pack/plugins/monitoring/server/lib/kibana/get_kibana_info.ts b/x-pack/plugins/monitoring/server/lib/kibana/get_kibana_info.ts index 81a653d370c0b1..4116e9e5b86acc 100644 --- a/x-pack/plugins/monitoring/server/lib/kibana/get_kibana_info.ts +++ b/x-pack/plugins/monitoring/server/lib/kibana/get_kibana_info.ts @@ -12,6 +12,8 @@ import { checkParam, MissingRequiredError } from '../error_missing_required'; import { calculateAvailability } from '../calculate_availability'; import { LegacyRequest } from '../../types'; import { ElasticsearchResponse } from '../../../common/types/es'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; import { buildKibanaInfo } from './build_kibana_info'; export function handleResponse(resp: ElasticsearchResponse) { @@ -32,13 +34,16 @@ export function handleResponse(resp: ElasticsearchResponse) { export function getKibanaInfo( req: LegacyRequest, - kbnIndexPattern: string, { clusterUuid, kibanaUuid }: { clusterUuid: string; kibanaUuid: string } ) { - checkParam(kbnIndexPattern, 'kbnIndexPattern in getKibanaInfo'); - + const moduleType = 'kibana'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + }); const params = { - index: kbnIndexPattern, + index: indexPatterns, size: 1, ignore_unavailable: true, filter_path: [ diff --git a/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas.ts b/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas.ts index 6e55bf83bbd02c..a476baa9c45d2b 100644 --- a/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas.ts +++ b/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas.ts @@ -15,6 +15,8 @@ import { calculateAvailability } from '../calculate_availability'; // @ts-ignore import { KibanaMetric } from '../metrics'; import { LegacyRequest } from '../../types'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; import { ElasticsearchResponse, ElasticsearchResponseHit } from '../../../common/types/es'; import { KibanaInfo, buildKibanaInfo } from './build_kibana_info'; @@ -52,24 +54,28 @@ interface Kibana { * - requests * - response times */ -export async function getKibanas( - req: LegacyRequest, - kbnIndexPattern: string, - { clusterUuid }: { clusterUuid: string } -) { - checkParam(kbnIndexPattern, 'kbnIndexPattern in getKibanas'); - +export async function getKibanas(req: LegacyRequest, { clusterUuid }: { clusterUuid: string }) { const config = req.server.config(); const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); - + const moduleType = 'kibana'; + const type = 'kibana_stats'; + const dataset = 'stats'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const params = { - index: kbnIndexPattern, + index: indexPatterns, size: config.get('monitoring.ui.max_bucket_size'), ignore_unavailable: true, body: { query: createQuery({ - types: ['kibana_stats', 'stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.ts b/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.ts index 5326976ec99acf..883ca17c98c5bd 100644 --- a/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.ts @@ -7,9 +7,10 @@ import { chain, find } from 'lodash'; import { LegacyRequest, Cluster, Bucket } from '../../types'; -import { checkParam } from '../error_missing_required'; import { createQuery } from '../create_query'; import { KibanaClusterMetric } from '../metrics'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; /* * Get high-level info for Kibanas in a set of clusters @@ -24,28 +25,34 @@ import { KibanaClusterMetric } from '../metrics'; * - number of instances * - combined health */ -export function getKibanasForClusters( - req: LegacyRequest, - kbnIndexPattern: string, - clusters: Cluster[] -) { - checkParam(kbnIndexPattern, 'kbnIndexPattern in kibana/getKibanasForClusters'); - +export function getKibanasForClusters(req: LegacyRequest, clusters: Cluster[], ccs: string) { const config = req.server.config(); const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; + const moduleType = 'kibana'; + const type = 'kibana_stats'; + const dataset = 'stats'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType, + dataset, + ccs, + }); + return Promise.all( clusters.map((cluster) => { const clusterUuid = cluster.elasticsearch?.cluster?.id ?? cluster.cluster_uuid; const metric = KibanaClusterMetric.getMetricFields(); const params = { - index: kbnIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, body: { query: createQuery({ - types: ['stats', 'kibana_stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/logs/init_infra_source.ts b/x-pack/plugins/monitoring/server/lib/logs/init_infra_source.ts index 727e47b62bc922..72a794db76183a 100644 --- a/x-pack/plugins/monitoring/server/lib/logs/init_infra_source.ts +++ b/x-pack/plugins/monitoring/server/lib/logs/init_infra_source.ts @@ -13,7 +13,7 @@ import { InfraPluginSetup } from '../../../../infra/server'; export const initInfraSource = (config: MonitoringConfig, infraPlugin: InfraPluginSetup) => { if (infraPlugin) { - const filebeatIndexPattern = prefixIndexPattern(config, config.ui.logs.index, '*', true); + const filebeatIndexPattern = prefixIndexPattern(config, config.ui.logs.index, '*'); infraPlugin.defineInternalSourceConfiguration(INFRA_SOURCE_ID, { name: 'Elastic Stack Logs', logIndices: { diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_cluster_status.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_cluster_status.ts index dfd1eaa1550691..308a750f6ef020 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_cluster_status.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_cluster_status.ts @@ -6,7 +6,6 @@ */ import { get } from 'lodash'; -import { checkParam } from '../error_missing_required'; import { getLogstashForClusters } from './get_logstash_for_clusters'; import { LegacyRequest } from '../../types'; @@ -16,18 +15,12 @@ import { LegacyRequest } from '../../types'; * Shared functionality between the different routes. * * @param {Object} req The incoming request. - * @param {String} lsIndexPattern The Logstash pattern to query for the current time range. * @param {String} clusterUuid The cluster UUID for the associated Elasticsearch cluster. * @returns {Promise} The cluster status object. */ -export function getClusterStatus( - req: LegacyRequest, - lsIndexPattern: string, - { clusterUuid }: { clusterUuid: string } -) { - checkParam(lsIndexPattern, 'lsIndexPattern in logstash/getClusterStatus'); +export function getClusterStatus(req: LegacyRequest, { clusterUuid }: { clusterUuid: string }) { const clusters = [{ cluster_uuid: clusterUuid }]; - return getLogstashForClusters(req, lsIndexPattern, clusters).then((clusterStatus) => + return getLogstashForClusters(req, clusters).then((clusterStatus) => get(clusterStatus, '[0].stats') ); } diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.ts index 20e611a5ee3dad..a0be8efe5ebdf7 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.ts @@ -8,9 +8,10 @@ import { get } from 'lodash'; import { LegacyRequest, Cluster, Bucket } from '../../types'; import { LOGSTASH } from '../../../common/constants'; -import { checkParam } from '../error_missing_required'; import { createQuery } from '../create_query'; import { LogstashClusterMetric } from '../metrics'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; const { MEMORY, PERSISTED } = LOGSTASH.QUEUE_TYPES; @@ -38,25 +39,35 @@ const getQueueTypes = (queueBuckets: Array + clusters: Array<{ cluster_uuid: string } | Cluster>, + ccs?: string ) { - checkParam(lsIndexPattern, 'lsIndexPattern in logstash/getLogstashForClusters'); - const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; const config = req.server.config(); + const dataset = 'node_stats'; + const type = 'logstash_stats'; + const moduleType = 'logstash'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: ccs || req.payload.ccs, + moduleType, + dataset, + }); + return Promise.all( clusters.map((cluster) => { const clusterUuid = get(cluster, 'elasticsearch.cluster.id', cluster.cluster_uuid); const params = { - index: lsIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, body: { query: createQuery({ - types: ['node_stats', 'logstash_stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.test.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.test.ts index cee6c144c866ea..a43eb9a7cd09fe 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.test.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.test.ts @@ -18,6 +18,18 @@ interface HitParams { value?: string; } +jest.mock('../../static_globals', () => ({ + Globals: { + app: { + config: { + ui: { + ccs: { enabled: true }, + }, + }, + }, + }, +})); + // deletes, adds, or updates the properties based on a default object function createResponseObjHit(params?: HitParams[]): ElasticsearchResponseHit { const defaultResponseObj: ElasticsearchResponseHit = { @@ -189,7 +201,11 @@ describe('get_logstash_info', () => { then: jest.fn(), }); const req = { + payload: {}, server: { + config: () => ({ + get: () => undefined, + }), plugins: { elasticsearch: { getCluster: () => ({ @@ -199,7 +215,8 @@ describe('get_logstash_info', () => { }, }, } as unknown as LegacyRequest; - await getNodeInfo(req, '.monitoring-logstash-*', { + + await getNodeInfo(req, { clusterUuid: STANDALONE_CLUSTER_CLUSTER_UUID, logstashUuid: 'logstash_uuid', }); diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.ts index ebd1128dce3646..92e2a836e08ff8 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_node_info.ts @@ -6,12 +6,14 @@ */ import { merge } from 'lodash'; -import { checkParam, MissingRequiredError } from '../error_missing_required'; +import { MissingRequiredError } from '../error_missing_required'; import { calculateAvailability } from '../calculate_availability'; import { LegacyRequest } from '../../types'; import { ElasticsearchResponse } from '../../../common/types/es'; import { STANDALONE_CLUSTER_CLUSTER_UUID } from '../../../common/constants'; import { standaloneClusterFilter } from '../standalone_clusters/standalone_cluster_query_filter'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; export function handleResponse(resp: ElasticsearchResponse) { const legacyStats = resp.hits?.hits[0]?._source?.logstash_stats; @@ -33,18 +35,25 @@ export function handleResponse(resp: ElasticsearchResponse) { export function getNodeInfo( req: LegacyRequest, - lsIndexPattern: string, { clusterUuid, logstashUuid }: { clusterUuid: string; logstashUuid: string } ) { - checkParam(lsIndexPattern, 'lsIndexPattern in getNodeInfo'); const isStandaloneCluster = clusterUuid === STANDALONE_CLUSTER_CLUSTER_UUID; const clusterFilter = isStandaloneCluster ? standaloneClusterFilter : { term: { cluster_uuid: clusterUuid } }; + const dataset = 'node_stats'; + const moduleType = 'logstash'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); + const params = { - index: lsIndexPattern, + index: indexPatterns, size: 1, ignore_unavailable: true, filter_path: [ diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_nodes.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_nodes.ts index a25b57ab445e32..d7c490b1d2fd6d 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_nodes.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_nodes.ts @@ -6,12 +6,13 @@ */ import moment from 'moment'; -import { checkParam } from '../error_missing_required'; import { createQuery } from '../create_query'; import { calculateAvailability } from '../calculate_availability'; import { LogstashMetric } from '../metrics'; import { LegacyRequest } from '../../types'; import { ElasticsearchResponse } from '../../../common/types/es'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; interface Logstash { jvm?: { @@ -64,28 +65,35 @@ interface Logstash { * - events * - config reloads */ -export async function getNodes( - req: LegacyRequest, - lsIndexPattern: string, - { clusterUuid }: { clusterUuid: string } -) { - checkParam(lsIndexPattern, 'lsIndexPattern in getNodes'); +export async function getNodes(req: LegacyRequest, { clusterUuid }: { clusterUuid: string }) { + const dataset = 'node_stats'; + const type = 'logstash_stats'; + const moduleType = 'logstash'; + + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const config = req.server.config(); const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); const params = { - index: lsIndexPattern, + index: indexPatterns, size: config.get('monitoring.ui.max_bucket_size'), // FIXME ignore_unavailable: true, body: { query: createQuery({ + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, clusterUuid, metric: LogstashMetric.getMetricFields(), - types: ['node_stats', 'logstash_stats'], }), collapse: { field: 'logstash_stats.logstash.uuid', diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.ts index f8f993874a1814..42c6a684234c45 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.ts @@ -42,7 +42,6 @@ import { interface GetPaginatedPipelinesParams { req: LegacyRequest; - lsIndexPattern: string; clusterUuid: string; logstashUuid?: string; metrics: { @@ -55,7 +54,6 @@ interface GetPaginatedPipelinesParams { } export async function getPaginatedPipelines({ req, - lsIndexPattern, clusterUuid, logstashUuid, metrics, @@ -70,16 +68,15 @@ export async function getPaginatedPipelines({ const size = config.get('monitoring.ui.max_bucket_size') as unknown as number; let pipelines = await getLogstashPipelineIds({ req, - lsIndexPattern, clusterUuid, logstashUuid, size, }); // this is needed for sorting if (sortField === throughputMetric) { - pipelines = await getPaginatedThroughputData(pipelines, req, lsIndexPattern, throughputMetric); + pipelines = await getPaginatedThroughputData(pipelines, req, throughputMetric); } else if (sortField === nodesCountMetric) { - pipelines = await getPaginatedNodesData(pipelines, req, lsIndexPattern, nodesCountMetric); + pipelines = await getPaginatedNodesData(pipelines, req, nodesCountMetric); } const filteredPipelines = filter(pipelines, queryText, ['id']); // We only support filtering by id right now @@ -91,7 +88,6 @@ export async function getPaginatedPipelines({ const response = { pipelines: await getPipelines({ req, - lsIndexPattern, pipelines: pageOfPipelines, throughputMetric, nodesCountMetric, @@ -131,9 +127,10 @@ function processPipelinesAPIResponse( async function getPaginatedThroughputData( pipelines: Pipeline[], req: LegacyRequest, - lsIndexPattern: string, throughputMetric: PipelineThroughputMetricKey ): Promise { + const dataset = 'node_stats'; + const moduleType = 'logstash'; const metricSeriesData: any = Object.values( await Promise.all( pipelines.map((pipeline) => { @@ -141,22 +138,19 @@ async function getPaginatedThroughputData( try { const data = await getMetrics( req, - lsIndexPattern, + moduleType, [throughputMetric], [ { bool: { should: [ + { term: { 'data_stream.dataset': `${moduleType}.${dataset}` } }, + { term: { 'metricset.name': dataset } }, { term: { type: 'logstash_stats', }, }, - { - term: { - 'metricset.name': 'node_stats', - }, - }, ], }, }, @@ -197,20 +191,22 @@ async function getPaginatedThroughputData( async function getPaginatedNodesData( pipelines: Pipeline[], req: LegacyRequest, - lsIndexPattern: string, nodesCountMetric: PipelineNodeCountMetricKey ): Promise { + const dataset = 'node_stats'; + const moduleType = 'logstash'; const pipelineWithMetrics = cloneDeep(pipelines); const metricSeriesData = await getMetrics( req, - lsIndexPattern, + moduleType, [nodesCountMetric], [ { bool: { should: [ + { term: { 'data_stream.dataset': `${moduleType}.${dataset}` } }, + { term: { 'metricset.name': dataset } }, { term: { type: 'logstash_stats' } }, - { term: { 'metricset.name': 'node_stats' } }, ], }, }, @@ -231,29 +227,17 @@ async function getPaginatedNodesData( async function getPipelines({ req, - lsIndexPattern, pipelines, throughputMetric, nodesCountMetric, }: { req: LegacyRequest; - lsIndexPattern: string; pipelines: Pipeline[]; throughputMetric: PipelineThroughputMetricKey; nodesCountMetric: PipelineNodeCountMetricKey; }): Promise { - const throughputPipelines = await getThroughputPipelines( - req, - lsIndexPattern, - pipelines, - throughputMetric - ); - const nodeCountPipelines = await getNodePipelines( - req, - lsIndexPattern, - pipelines, - nodesCountMetric - ); + const throughputPipelines = await getThroughputPipelines(req, pipelines, throughputMetric); + const nodeCountPipelines = await getNodePipelines(req, pipelines, nodesCountMetric); const finalPipelines = pipelines.map(({ id }) => { const matchThroughputPipeline = throughputPipelines.find((p) => p.id === id); const matchNodesCountPipeline = nodeCountPipelines.find((p) => p.id === id); @@ -276,32 +260,30 @@ async function getPipelines({ async function getThroughputPipelines( req: LegacyRequest, - lsIndexPattern: string, pipelines: Pipeline[], throughputMetric: string ): Promise { + const dataset = 'node_stats'; + const moduleType = 'logstash'; const metricsResponse = await Promise.all( pipelines.map((pipeline) => { return new Promise(async (resolve, reject) => { try { const data = await getMetrics( req, - lsIndexPattern, + moduleType, [throughputMetric], [ { bool: { should: [ + { term: { 'data_stream.dataset': `${moduleType}.${dataset}` } }, + { term: { 'metricset.name': dataset } }, { term: { type: 'logstash_stats', }, }, - { - term: { - 'metricset.name': 'node_stats', - }, - }, ], }, }, @@ -322,20 +304,22 @@ async function getThroughputPipelines( async function getNodePipelines( req: LegacyRequest, - lsIndexPattern: string, pipelines: Pipeline[], nodesCountMetric: string ): Promise { + const moduleType = 'logstash'; + const dataset = 'node_stats'; const metricData = await getMetrics( req, - lsIndexPattern, + moduleType, [nodesCountMetric], [ { bool: { should: [ + { term: { 'data_stream.dataset': `${moduleType}.${dataset}` } }, + { term: { 'metricset.name': dataset } }, { term: { type: 'logstash_stats' } }, - { term: { 'metricset.name': 'node_stats' } }, ], }, }, diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline.ts index 1e8eb94df4836b..16a96b132483f7 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline.ts @@ -7,7 +7,6 @@ import boom from '@hapi/boom'; import { get } from 'lodash'; -import { checkParam } from '../error_missing_required'; import { getPipelineStateDocument } from './get_pipeline_state_document'; import { getPipelineStatsAggregation } from './get_pipeline_stats_aggregation'; import { calculateTimeseriesInterval } from '../calculate_timeseries_interval'; @@ -122,13 +121,10 @@ export function _enrichStateWithStatsAggregation( export async function getPipeline( req: LegacyRequest, config: { get: (key: string) => string | undefined }, - lsIndexPattern: string, clusterUuid: string, pipelineId: string, version: PipelineVersion ) { - checkParam(lsIndexPattern, 'lsIndexPattern in getPipeline'); - // Determine metrics' timeseries interval based on version's timespan const minIntervalSeconds = Math.max(Number(config.get('monitoring.ui.min_interval_seconds')), 30); const timeseriesInterval = calculateTimeseriesInterval( @@ -140,14 +136,12 @@ export async function getPipeline( const [stateDocument, statsAggregation] = await Promise.all([ getPipelineStateDocument({ req, - logstashIndexPattern: lsIndexPattern, clusterUuid, pipelineId, version, }), getPipelineStatsAggregation({ req, - logstashIndexPattern: lsIndexPattern, timeseriesInterval, clusterUuid, pipelineId, diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_ids.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_ids.ts index c9b7a3adfc18e7..7654ed551b63b4 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_ids.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_ids.ts @@ -10,20 +10,22 @@ import { get } from 'lodash'; import { LegacyRequest, Bucket, Pipeline } from '../../types'; import { createQuery } from '../create_query'; import { LogstashMetric } from '../metrics'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; interface GetLogstashPipelineIdsParams { req: LegacyRequest; - lsIndexPattern: string; clusterUuid: string; size: number; logstashUuid?: string; + ccs?: string; } export async function getLogstashPipelineIds({ req, - lsIndexPattern, clusterUuid, logstashUuid, size, + ccs, }: GetLogstashPipelineIdsParams): Promise { const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); @@ -33,8 +35,17 @@ export async function getLogstashPipelineIds({ filters.push({ term: { 'logstash_stats.logstash.uuid': logstashUuid } }); } + const dataset = 'node_stats'; + const moduleType = 'logstash'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: ccs || req.payload.ccs, + moduleType, + dataset, + }); + const params = { - index: lsIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, filter_path: ['aggregations.nest.id.buckets', 'aggregations.nest_mb.id.buckets'], diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_state_document.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_state_document.ts index f62d4de1219ecf..6c2504efe29ff3 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_state_document.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_state_document.ts @@ -9,20 +9,29 @@ import { createQuery } from '../create_query'; import { LogstashMetric } from '../metrics'; import { LegacyRequest, PipelineVersion } from '../../types'; import { ElasticsearchResponse } from '../../../common/types/es'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; export async function getPipelineStateDocument({ req, - logstashIndexPattern, clusterUuid, pipelineId, version, }: { req: LegacyRequest; - logstashIndexPattern: string; clusterUuid: string; pipelineId: string; version: PipelineVersion; }) { + const dataset = 'node'; + const type = 'logstash_state'; + const moduleType = 'logstash'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const { callWithRequest } = req.server.plugins?.elasticsearch.getCluster('monitoring'); const filters = [ { term: { 'logstash_state.pipeline.id': pipelineId } }, @@ -35,14 +44,16 @@ export async function getPipelineStateDocument({ // This is important because a user may pick a very narrow time picker window. If we were to use a start/end value // that could result in us being unable to render the graph // Use the logstash_stats documents to determine whether the instance is up/down - types: ['logstash_state', 'node'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, metric: LogstashMetric.getMetricFields(), clusterUuid, filters, }); const params = { - index: logstashIndexPattern, + index: indexPatterns, size: 1, ignore_unavailable: true, body: { diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.ts index 70f856b25f8c85..31cfa4b4a02911 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.ts @@ -6,8 +6,10 @@ */ import { LegacyRequest, PipelineVersion } from '../../types'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; import { createQuery } from '../create_query'; import { LogstashMetric } from '../metrics'; +import { Globals } from '../../static_globals'; function scalarCounterAggregation( field: string, @@ -111,16 +113,23 @@ function createScopedAgg(pipelineId: string, pipelineHash: string, maxBucketSize function fetchPipelineLatestStats( query: object, - logstashIndexPattern: string, pipelineId: string, version: PipelineVersion, maxBucketSize: string, callWithRequest: any, req: LegacyRequest ) { + const dataset = 'node_stats'; + const moduleType = 'logstash'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const pipelineAggregation = createScopedAgg(pipelineId, version.hash, maxBucketSize); const params = { - index: logstashIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, filter_path: [ @@ -149,14 +158,12 @@ function fetchPipelineLatestStats( export function getPipelineStatsAggregation({ req, - logstashIndexPattern, timeseriesInterval, clusterUuid, pipelineId, version, }: { req: LegacyRequest; - logstashIndexPattern: string; timeseriesInterval: number; clusterUuid: string; pipelineId: string; @@ -197,8 +204,14 @@ export function getPipelineStatsAggregation({ const start = version.lastSeen - timeseriesInterval * 1000; const end = version.lastSeen; + const dataset = 'node_stats'; + const type = 'logstash_stats'; + const moduleType = 'logstash'; + const query = createQuery({ - types: ['node_stats', 'logstash_stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, metric: LogstashMetric.getMetricFields(), @@ -210,7 +223,6 @@ export function getPipelineStatsAggregation({ return fetchPipelineLatestStats( query, - logstashIndexPattern, pipelineId, version, // @ts-ignore not undefined, need to get correct config diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_versions.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_versions.ts index 0dbbf26b331e2f..eecc4388f89470 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_versions.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_versions.ts @@ -8,7 +8,8 @@ import { get, orderBy } from 'lodash'; import { createQuery } from '../create_query'; import { LogstashMetric } from '../metrics'; -import { checkParam } from '../error_missing_required'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; +import { Globals } from '../../static_globals'; import { LegacyRequest, PipelineVersion } from '../../types'; import { mergePipelineVersions } from './merge_pipeline_versions'; @@ -61,17 +62,23 @@ const createScopedAgg = (pipelineId: string, maxBucketSize: number) => { function fetchPipelineVersions({ req, - lsIndexPattern, clusterUuid, pipelineId, }: { req: LegacyRequest; - lsIndexPattern: string; clusterUuid: string; pipelineId: string; }) { + const dataset = 'node_stats'; + const type = 'logstash_stats'; + const moduleType = 'logstash'; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const config = req.server.config(); - checkParam(lsIndexPattern, 'logstashIndexPattern in getPipelineVersions'); const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); const filters = [ @@ -105,7 +112,9 @@ function fetchPipelineVersions({ }, ]; const query = createQuery({ - types: ['node_stats', 'logstash_stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, metric: LogstashMetric.getMetricFields(), clusterUuid, filters, @@ -121,7 +130,7 @@ function fetchPipelineVersions({ }; const params = { - index: lsIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, body: { @@ -159,7 +168,6 @@ export function _handleResponse(response: any) { export async function getPipelineVersions(args: { req: LegacyRequest; - lsIndexPattern: string; clusterUuid: string; pipelineId: string; }) { diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.ts index 75b82787ef3d0d..5605b23b8988bd 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.ts @@ -7,7 +7,6 @@ import boom from '@hapi/boom'; import { get } from 'lodash'; -import { checkParam } from '../error_missing_required'; import { getPipelineStateDocument } from './get_pipeline_state_document'; import { getPipelineVertexStatsAggregation } from './get_pipeline_vertex_stats_aggregation'; import { calculateTimeseriesInterval } from '../calculate_timeseries_interval'; @@ -137,14 +136,11 @@ export function _enrichVertexStateWithStatsAggregation( export async function getPipelineVertex( req: LegacyRequest, config: { get: (key: string) => string | undefined }, - lsIndexPattern: string, clusterUuid: string, pipelineId: string, version: PipelineVersion, vertexId: string ) { - checkParam(lsIndexPattern, 'lsIndexPattern in getPipeline'); - // Determine metrics' timeseries interval based on version's timespan const minIntervalSeconds = Math.max(Number(config.get('monitoring.ui.min_interval_seconds')), 30); const timeseriesInterval = calculateTimeseriesInterval( @@ -156,14 +152,12 @@ export async function getPipelineVertex( const [stateDocument, statsAggregation] = await Promise.all([ getPipelineStateDocument({ req, - logstashIndexPattern: lsIndexPattern, clusterUuid, pipelineId, version, }), getPipelineVertexStatsAggregation({ req, - logstashIndexPattern: lsIndexPattern, timeSeriesIntervalInSeconds: timeseriesInterval, clusterUuid, pipelineId, diff --git a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.ts b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.ts index bfd803f0a86d58..8b26f5d44855b0 100644 --- a/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.ts +++ b/x-pack/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.ts @@ -6,8 +6,10 @@ */ import { LegacyRequest, PipelineVersion } from '../../types'; +import { getNewIndexPatterns } from '../cluster/get_index_patterns'; import { createQuery } from '../create_query'; import { LogstashMetric } from '../metrics'; +import { Globals } from '../../static_globals'; function scalarCounterAggregation( field: string, @@ -147,7 +149,6 @@ function createTimeSeriesAgg(timeSeriesIntervalInSeconds: number, ...aggsList: o function fetchPipelineVertexTimeSeriesStats({ query, - logstashIndexPattern, pipelineId, version, vertexId, @@ -157,7 +158,6 @@ function fetchPipelineVertexTimeSeriesStats({ req, }: { query: object; - logstashIndexPattern: string; pipelineId: string; version: PipelineVersion; vertexId: string; @@ -174,8 +174,14 @@ function fetchPipelineVertexTimeSeriesStats({ }), }; + const indexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType: 'logstash', + }); + const params = { - index: logstashIndexPattern, + index: indexPatterns, size: 0, ignore_unavailable: true, filter_path: [ @@ -202,7 +208,6 @@ function fetchPipelineVertexTimeSeriesStats({ export function getPipelineVertexStatsAggregation({ req, - logstashIndexPattern, timeSeriesIntervalInSeconds, clusterUuid, pipelineId, @@ -210,7 +215,6 @@ export function getPipelineVertexStatsAggregation({ vertexId, }: { req: LegacyRequest; - logstashIndexPattern: string; timeSeriesIntervalInSeconds: number; clusterUuid: string; pipelineId: string; @@ -252,8 +256,14 @@ export function getPipelineVertexStatsAggregation({ const start = version.firstSeen; const end = version.lastSeen; + const moduleType = 'logstash'; + const dataset = 'node_stats'; + const type = 'logstash_stats'; + const query = createQuery({ - types: ['node_stats', 'logstash_stats'], + type, + dsDataset: `${moduleType}.${dataset}`, + metricset: dataset, start, end, metric: LogstashMetric.getMetricFields(), @@ -265,7 +275,6 @@ export function getPipelineVertexStatsAggregation({ return fetchPipelineVertexTimeSeriesStats({ query, - logstashIndexPattern, pipelineId, version, vertexId, diff --git a/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.test.js b/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.test.js index 1029f00455c690..bbcedf7fdd33d2 100644 --- a/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.test.js +++ b/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.test.js @@ -34,9 +34,6 @@ const mockReq = ( if (prop === 'server.uuid') { return 'kibana-1234'; } - if (prop === 'monitoring.ui.metricbeat.index') { - return 'metricbeat-*'; - } }), }; }, diff --git a/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.ts b/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.ts index eda9315842040b..463ccf547d5dbc 100644 --- a/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.ts +++ b/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.ts @@ -291,16 +291,6 @@ function getUuidBucketName(productName: string) { } } -function matchesMetricbeatIndex(metricbeatIndex: string, index: string) { - if (index.includes(metricbeatIndex)) { - return true; - } - if (metricbeatIndex.includes('*')) { - return new RegExp(metricbeatIndex).test(index); - } - return false; -} - function isBeatFromAPM(bucket: Bucket) { const beatType = get(bucket, 'single_type.beat_type'); if (!beatType) { @@ -443,7 +433,6 @@ export const getCollectionStatus = async ( ) => { const config = req.server.config(); const kibanaUuid = config.get('server.uuid'); - const metricbeatIndex = config.get('monitoring.ui.metricbeat.index')!; const size = config.get('monitoring.ui.max_bucket_size'); const hasPermissions = await hasNecessaryPermissions(req); @@ -484,11 +473,6 @@ export const getCollectionStatus = async ( if (bucket.key.includes(token)) { return true; } - if (matchesMetricbeatIndex(metricbeatIndex, bucket.key)) { - if (get(bucket, `${uuidBucketName}.buckets`, []).length) { - return true; - } - } return false; }); @@ -512,9 +496,7 @@ export const getCollectionStatus = async ( // If there is a single bucket, then they are fully migrated or fully on the internal collector else if (indexBuckets.length === 1) { const singleIndexBucket = indexBuckets[0]; - const isFullyMigrated = - singleIndexBucket.key.includes(METRICBEAT_INDEX_NAME_UNIQUE_TOKEN) || - matchesMetricbeatIndex(metricbeatIndex, singleIndexBucket.key); + const isFullyMigrated = singleIndexBucket.key.includes(METRICBEAT_INDEX_NAME_UNIQUE_TOKEN); const map = isFullyMigrated ? fullyMigratedUuidsMap : internalCollectorsUuidsMap; const uuidBuckets = get(singleIndexBucket, `${uuidBucketName}.buckets`, []); @@ -594,8 +576,7 @@ export const getCollectionStatus = async ( for (const indexBucket of indexBuckets) { const isFullyMigrated = considerAllInstancesMigrated || - indexBucket.key.includes(METRICBEAT_INDEX_NAME_UNIQUE_TOKEN) || - matchesMetricbeatIndex(metricbeatIndex, indexBucket.key); + indexBucket.key.includes(METRICBEAT_INDEX_NAME_UNIQUE_TOKEN); const map = isFullyMigrated ? fullyMigratedUuidsMap : internalCollectorsUuidsMap; const otherMap = !isFullyMigrated ? fullyMigratedUuidsMap : internalCollectorsUuidsMap; diff --git a/x-pack/plugins/monitoring/server/lib/standalone_clusters/has_standalone_clusters.ts b/x-pack/plugins/monitoring/server/lib/standalone_clusters/has_standalone_clusters.ts index 4aacc6d14d0f99..1859c66e9e7132 100644 --- a/x-pack/plugins/monitoring/server/lib/standalone_clusters/has_standalone_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/standalone_clusters/has_standalone_clusters.ts @@ -9,8 +9,22 @@ import moment from 'moment'; import { get } from 'lodash'; import { LegacyRequest } from '../../types'; import { standaloneClusterFilter } from './'; +import { Globals } from '../../static_globals'; +import { getLegacyIndexPattern, getNewIndexPatterns } from '../cluster/get_index_patterns'; -export async function hasStandaloneClusters(req: LegacyRequest, indexPatterns: string[]) { +export async function hasStandaloneClusters(req: LegacyRequest, ccs: string) { + const lsIndexPatterns = getNewIndexPatterns({ + config: Globals.app.config, + moduleType: 'logstash', + ccs, + }); + // use legacy because no integration exists for beats + const beatsIndexPatterns = getLegacyIndexPattern({ + moduleType: 'beats', + config: Globals.app.config, + ccs, + }); + const indexPatterns = [lsIndexPatterns, beatsIndexPatterns]; const indexPatternList = indexPatterns.reduce((list, patterns) => { list.push(...patterns.split(',')); return list; @@ -28,7 +42,12 @@ export async function hasStandaloneClusters(req: LegacyRequest, indexPatterns: s }, { terms: { - 'event.dataset': ['logstash.node.stats', 'logstash.node', 'beat.stats', 'beat.state'], + 'metricset.name': ['node', 'node_stats', 'stats', 'state'], + }, + }, + { + terms: { + 'data_stream.dataset': ['node', 'node_stats', 'stats', 'state'], }, }, ], diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/apm/_get_apm_cluster_status.js b/x-pack/plugins/monitoring/server/routes/api/v1/apm/_get_apm_cluster_status.js index a28312de78af0b..bd0198ffcc3b24 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/apm/_get_apm_cluster_status.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/apm/_get_apm_cluster_status.js @@ -7,9 +7,9 @@ import { getApmsForClusters } from '../../../../lib/apm/get_apms_for_clusters'; -export const getApmClusterStatus = (req, apmIndexPattern, { clusterUuid }) => { +export const getApmClusterStatus = (req, { clusterUuid }) => { const clusters = [{ cluster_uuid: clusterUuid }]; - return getApmsForClusters(req, apmIndexPattern, clusters).then((apms) => { + return getApmsForClusters(req, clusters).then((apms) => { const [{ stats, config }] = apms; return { ...stats, diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/apm/instance.js b/x-pack/plugins/monitoring/server/routes/api/v1/apm/instance.js index a0b00167101fe4..4fa0de7d399d99 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/apm/instance.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/apm/instance.js @@ -47,9 +47,7 @@ export function apmInstanceRoute(server) { try { const [metrics, apmSummary] = await Promise.all([ - getMetrics(req, apmIndexPattern, metricSet, [ - { term: { 'beats_stats.beat.uuid': apmUuid } }, - ]), + getMetrics(req, 'beats', metricSet, [{ term: { 'beats_stats.beat.uuid': apmUuid } }]), getApmInfo(req, apmIndexPattern, { clusterUuid, apmUuid }), ]); diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/apm/overview.js b/x-pack/plugins/monitoring/server/routes/api/v1/apm/overview.js index ea7f3f41b842e8..b773cfd7b0fb97 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/apm/overview.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/apm/overview.js @@ -6,12 +6,10 @@ */ import { schema } from '@kbn/config-schema'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { getMetrics } from '../../../../lib/details/get_metrics'; import { metricSet } from './metric_set_overview'; import { handleError } from '../../../../lib/errors'; import { getApmClusterStatus } from './_get_apm_cluster_status'; -import { INDEX_PATTERN_BEATS } from '../../../../../common/constants'; export function apmOverviewRoute(server) { server.route({ @@ -33,9 +31,7 @@ export function apmOverviewRoute(server) { }, async handler(req) { const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; - const apmIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs); const showCgroupMetrics = config.get('monitoring.ui.container.apm.enabled'); if (showCgroupMetrics) { @@ -45,8 +41,8 @@ export function apmOverviewRoute(server) { try { const [stats, metrics] = await Promise.all([ - getApmClusterStatus(req, apmIndexPattern, { clusterUuid }), - getMetrics(req, apmIndexPattern, metricSet), + getApmClusterStatus(req, { clusterUuid }), + getMetrics(req, 'beats', metricSet), ]); return { diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/beats/beat_detail.js b/x-pack/plugins/monitoring/server/routes/api/v1/beats/beat_detail.js index 851380fede77dc..d9454b7ae62cf6 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/beats/beat_detail.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/beats/beat_detail.js @@ -49,9 +49,7 @@ export function beatsDetailRoute(server) { try { const [summary, metrics] = await Promise.all([ getBeatSummary(req, beatsIndexPattern, summaryOptions), - getMetrics(req, beatsIndexPattern, metricSet, [ - { term: { 'beats_stats.beat.uuid': beatUuid } }, - ]), + getMetrics(req, 'beats', metricSet, [{ term: { 'beats_stats.beat.uuid': beatUuid } }]), ]); return { diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/beats/overview.js b/x-pack/plugins/monitoring/server/routes/api/v1/beats/overview.js index 4abf46b3ad1cee..12349c8b855712 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/beats/overview.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/beats/overview.js @@ -41,7 +41,7 @@ export function beatsOverviewRoute(server) { const [latest, stats, metrics] = await Promise.all([ getLatestStats(req, beatsIndexPattern, clusterUuid), getStats(req, beatsIndexPattern, clusterUuid), - getMetrics(req, beatsIndexPattern, metricSet), + getMetrics(req, 'beats', metricSet), ]); return { diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr_shard.ts b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr_shard.ts index d07a6602224077..a1d76fe5ccd0d2 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr_shard.ts +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr_shard.ts @@ -13,9 +13,10 @@ import { handleError } from '../../../../lib/errors/handle_error'; import { prefixIndexPattern } from '../../../../../common/ccs_utils'; // @ts-ignore import { getMetrics } from '../../../../lib/details/get_metrics'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants'; import { ElasticsearchResponse } from '../../../../../common/types/es'; import { LegacyRequest } from '../../../../types'; +import { getNewIndexPatterns } from '../../../../lib/cluster/get_index_patterns'; +import { Globals } from '../../../../static_globals'; function getFormattedLeaderIndex(leaderIndex: string) { let leader = leaderIndex; @@ -98,27 +99,33 @@ export function ccrShardRoute(server: { route: (p: any) => void; config: () => { }, }, async handler(req: LegacyRequest) { - const config = server.config(); const index = req.params.index; const shardId = req.params.shardId; - const ccs = req.payload.ccs; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); + const moduleType = 'elasticsearch'; + const dataset = 'ccr'; + const esIndexPattern = getNewIndexPatterns({ + config: Globals.app.config, + ccs: req.payload.ccs, + moduleType, + dataset, + }); const filters = [ { bool: { should: [ + { term: { 'data_stream.dataset': { value: `${moduleType}.${dataset}` } } }, { term: { - type: { - value: 'ccr_stats', + 'metricset.name': { + value: dataset, }, }, }, { term: { - 'metricset.name': { - value: 'ccr', + type: { + value: 'ccr_stats', }, }, }, @@ -145,7 +152,7 @@ export function ccrShardRoute(server: { route: (p: any) => void; config: () => { const [metrics, ccrResponse]: [unknown, ElasticsearchResponse] = await Promise.all([ getMetrics( req, - esIndexPattern, + 'elasticsearch', [ { keys: ['ccr_sync_lag_time'], name: 'ccr_sync_lag_time' }, { keys: ['ccr_sync_lag_ops'], name: 'ccr_sync_lag_ops' }, diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index_detail.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index_detail.js index e99ae04ab282c6..6c6b1487da1064 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index_detail.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index_detail.js @@ -14,7 +14,6 @@ import { getShardAllocation, getShardStats } from '../../../../lib/elasticsearch import { handleError } from '../../../../lib/errors/handle_error'; import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { metricSet } from './metric_set_index_detail'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants'; import { getLogs } from '../../../../lib/logs/get_logs'; const { advanced: metricSetAdvanced, overview: metricSetOverview } = metricSet; @@ -42,12 +41,10 @@ export function esIndexRoute(server) { handler: async (req) => { try { const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; const indexUuid = req.params.id; const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); const filebeatIndexPattern = prefixIndexPattern( config, config.get('monitoring.ui.logs.index'), @@ -57,21 +54,21 @@ export function esIndexRoute(server) { const isAdvanced = req.payload.is_advanced; const metricSet = isAdvanced ? metricSetAdvanced : metricSetOverview; - const cluster = await getClusterStats(req, esIndexPattern, clusterUuid); + const cluster = await getClusterStats(req, clusterUuid); const showSystemIndices = true; // hardcode to true, because this could be a system index - const shardStats = await getShardStats(req, esIndexPattern, cluster, { + const shardStats = await getShardStats(req, cluster, { includeNodes: true, includeIndices: true, indexName: indexUuid, }); - const indexSummary = await getIndexSummary(req, esIndexPattern, shardStats, { + const indexSummary = await getIndexSummary(req, shardStats, { clusterUuid, indexUuid, start, end, }); - const metrics = await getMetrics(req, esIndexPattern, metricSet, [ + const metrics = await getMetrics(req, 'elasticsearch', metricSet, [ { term: { 'index_stats.index': indexUuid } }, ]); @@ -97,7 +94,7 @@ export function esIndexRoute(server) { stateUuid, showSystemIndices, }; - const shards = await getShardAllocation(req, esIndexPattern, allocationOptions); + const shards = await getShardAllocation(req, allocationOptions); logs = await getLogs(config, req, filebeatIndexPattern, { clusterUuid, diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/indices.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/indices.js index 76e769ac030bac..ea490a15471164 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/indices.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/indices.js @@ -10,8 +10,6 @@ import { getClusterStats } from '../../../../lib/cluster/get_cluster_stats'; import { getClusterStatus } from '../../../../lib/cluster/get_cluster_status'; import { getIndices } from '../../../../lib/elasticsearch/indices'; import { handleError } from '../../../../lib/errors/handle_error'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants'; import { getIndicesUnassignedShardStats } from '../../../../lib/elasticsearch/shards/get_indices_unassigned_shard_stats'; export function esIndicesRoute(server) { @@ -36,25 +34,14 @@ export function esIndicesRoute(server) { }, }, async handler(req) { - const config = server.config(); const { clusterUuid } = req.params; const { show_system_indices: showSystemIndices } = req.query; const { ccs } = req.payload; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); try { - const clusterStats = await getClusterStats(req, esIndexPattern, clusterUuid); - const indicesUnassignedShardStats = await getIndicesUnassignedShardStats( - req, - esIndexPattern, - clusterStats - ); - const indices = await getIndices( - req, - esIndexPattern, - showSystemIndices, - indicesUnassignedShardStats - ); + const clusterStats = await getClusterStats(req, clusterUuid, ccs); + const indicesUnassignedShardStats = await getIndicesUnassignedShardStats(req, clusterStats); + const indices = await getIndices(req, showSystemIndices, indicesUnassignedShardStats); return { clusterStatus: getClusterStatus(clusterStats, indicesUnassignedShardStats), diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ml_jobs.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ml_jobs.js index 5853cc3d6ee9d1..d27ec8ce3cc83e 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ml_jobs.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ml_jobs.js @@ -10,8 +10,6 @@ import { getClusterStats } from '../../../../lib/cluster/get_cluster_stats'; import { getClusterStatus } from '../../../../lib/cluster/get_cluster_status'; import { getMlJobs } from '../../../../lib/elasticsearch/get_ml_jobs'; import { handleError } from '../../../../lib/errors/handle_error'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants'; import { getIndicesUnassignedShardStats } from '../../../../lib/elasticsearch/shards/get_indices_unassigned_shard_stats'; export function mlJobRoute(server) { @@ -33,20 +31,12 @@ export function mlJobRoute(server) { }, }, async handler(req) { - const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); try { - const clusterStats = await getClusterStats(req, esIndexPattern, clusterUuid); - const indicesUnassignedShardStats = await getIndicesUnassignedShardStats( - req, - esIndexPattern, - clusterStats - ); - const rows = await getMlJobs(req, esIndexPattern); - + const clusterStats = await getClusterStats(req, clusterUuid); + const indicesUnassignedShardStats = await getIndicesUnassignedShardStats(req, clusterStats); + const rows = await getMlJobs(req); return { clusterStatus: getClusterStatus(clusterStats, indicesUnassignedShardStats), rows, diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js index 5f77d0394a4f19..86048ed1765abb 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js @@ -14,7 +14,6 @@ import { getMetrics } from '../../../../lib/details/get_metrics'; import { handleError } from '../../../../lib/errors/handle_error'; import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { metricSets } from './metric_set_node_detail'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants'; import { getLogs } from '../../../../lib/logs/get_logs'; const { advanced: metricSetAdvanced, overview: metricSetOverview } = metricSets; @@ -48,7 +47,6 @@ export function esNodeRoute(server) { const nodeUuid = req.params.nodeUuid; const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); const filebeatIndexPattern = prefixIndexPattern( config, config.get('monitoring.ui.logs.index'), @@ -75,25 +73,26 @@ export function esNodeRoute(server) { } try { - const cluster = await getClusterStats(req, esIndexPattern, clusterUuid); + const cluster = await getClusterStats(req, clusterUuid, ccs); const clusterState = get( cluster, 'cluster_state', get(cluster, 'elasticsearch.cluster.stats.state') ); - const shardStats = await getShardStats(req, esIndexPattern, cluster, { + + const shardStats = await getShardStats(req, cluster, { includeIndices: true, includeNodes: true, nodeUuid, }); - const nodeSummary = await getNodeSummary(req, esIndexPattern, clusterState, shardStats, { + const nodeSummary = await getNodeSummary(req, clusterState, shardStats, { clusterUuid, nodeUuid, start, end, }); - const metrics = await getMetrics(req, esIndexPattern, metricSet, [ + const metrics = await getMetrics(req, 'elasticsearch', metricSet, [ { term: { 'source_node.uuid': nodeUuid } }, ]); let logs; @@ -118,7 +117,7 @@ export function esNodeRoute(server) { stateUuid, showSystemIndices, }; - const shards = await getShardAllocation(req, esIndexPattern, allocationOptions); + const shards = await getShardAllocation(req, allocationOptions); shardAllocation = { shards, diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/nodes.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/nodes.js index 7ea2e6e1e14404..d6cff7ecd9ae9a 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/nodes.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/nodes.js @@ -11,8 +11,6 @@ import { getClusterStatus } from '../../../../lib/cluster/get_cluster_status'; import { getNodes } from '../../../../lib/elasticsearch/nodes'; import { getNodesShardCount } from '../../../../lib/elasticsearch/shards/get_nodes_shard_count'; import { handleError } from '../../../../lib/errors/handle_error'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants'; import { getPaginatedNodes } from '../../../../lib/elasticsearch/nodes/get_nodes/get_paginated_nodes'; import { LISTING_METRICS_NAMES } from '../../../../lib/elasticsearch/nodes/get_nodes/nodes_listing_metrics'; import { getIndicesUnassignedShardStats } from '../../../../lib/elasticsearch/shards/get_indices_unassigned_shard_stats'; @@ -45,25 +43,18 @@ export function esNodesRoute(server) { }, }, async handler(req) { - const config = server.config(); const { ccs, pagination, sort, queryText } = req.payload; const clusterUuid = req.params.clusterUuid; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); try { - const clusterStats = await getClusterStats(req, esIndexPattern, clusterUuid); - const nodesShardCount = await getNodesShardCount(req, esIndexPattern, clusterStats); - const indicesUnassignedShardStats = await getIndicesUnassignedShardStats( - req, - esIndexPattern, - clusterStats - ); + const clusterStats = await getClusterStats(req, clusterUuid); + const nodesShardCount = await getNodesShardCount(req, clusterStats); + const indicesUnassignedShardStats = await getIndicesUnassignedShardStats(req, clusterStats); const clusterStatus = getClusterStatus(clusterStats, indicesUnassignedShardStats); const metricSet = LISTING_METRICS_NAMES; const { pageOfNodes, totalNodeCount } = await getPaginatedNodes( req, - esIndexPattern, { clusterUuid }, metricSet, pagination, @@ -72,16 +63,11 @@ export function esNodesRoute(server) { { clusterStats, nodesShardCount, - } + }, + ccs ); - const nodes = await getNodes( - req, - esIndexPattern, - pageOfNodes, - clusterStats, - nodesShardCount - ); + const nodes = await getNodes(req, pageOfNodes, clusterStats, nodesShardCount); return { clusterStatus, nodes, totalNodeCount }; } catch (err) { throw handleError(err, req); diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/overview.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/overview.js index a84ca61a9396b0..6c74f9dd872d27 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/overview.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/overview.js @@ -13,10 +13,6 @@ import { getMetrics } from '../../../../lib/details/get_metrics'; import { handleError } from '../../../../lib/errors/handle_error'; import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { metricSet } from './metric_set_overview'; -import { - INDEX_PATTERN_ELASTICSEARCH, - INDEX_PATTERN_ELASTICSEARCH_ECS, -} from '../../../../../common/constants'; import { getLogs } from '../../../../lib/logs'; import { getIndicesUnassignedShardStats } from '../../../../lib/elasticsearch/shards/get_indices_unassigned_shard_stats'; @@ -40,22 +36,7 @@ export function esOverviewRoute(server) { }, async handler(req) { const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); - const esLegacyIndexPattern = prefixIndexPattern( - config, - INDEX_PATTERN_ELASTICSEARCH, - ccs, - true - ); - const esEcsIndexPattern = prefixIndexPattern( - config, - INDEX_PATTERN_ELASTICSEARCH_ECS, - ccs, - true - ); - const filebeatIndexPattern = prefixIndexPattern( config, config.get('monitoring.ui.logs.index'), @@ -68,21 +49,12 @@ export function esOverviewRoute(server) { try { const [clusterStats, metrics, shardActivity, logs] = await Promise.all([ - getClusterStats(req, esIndexPattern, clusterUuid), - getMetrics(req, esIndexPattern, metricSet), - getLastRecovery( - req, - esLegacyIndexPattern, - esEcsIndexPattern, - config.get('monitoring.ui.max_bucket_size') - ), + getClusterStats(req, clusterUuid), + getMetrics(req, 'elasticsearch', metricSet), + getLastRecovery(req, config.get('monitoring.ui.max_bucket_size')), getLogs(config, req, filebeatIndexPattern, { clusterUuid, start, end }), ]); - const indicesUnassignedShardStats = await getIndicesUnassignedShardStats( - req, - esIndexPattern, - clusterStats - ); + const indicesUnassignedShardStats = await getIndicesUnassignedShardStats(req, clusterStats); const result = { clusterStatus: getClusterStatus(clusterStats, indicesUnassignedShardStats), diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch_settings/check/internal_monitoring.ts b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch_settings/check/internal_monitoring.ts index eee6ba98e62c77..46626066005adf 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch_settings/check/internal_monitoring.ts +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch_settings/check/internal_monitoring.ts @@ -89,9 +89,9 @@ export function internalMonitoringCheckRoute(server: LegacyServer, npRoute: Rout const config = server.config(); const { ccs } = request.body; - const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs, true); - const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs, true); - const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs, true); + const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs); + const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs); + const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs); const indexCounts = await Promise.all([ checkLatestMonitoringIsLegacy(context, esIndexPattern), checkLatestMonitoringIsLegacy(context, kbnIndexPattern), diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/enterprise_search/overview.js b/x-pack/plugins/monitoring/server/routes/api/v1/enterprise_search/overview.js index b9bc0f49bc99d4..97ab818a829920 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/enterprise_search/overview.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/enterprise_search/overview.js @@ -6,11 +6,9 @@ */ import { schema } from '@kbn/config-schema'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { getMetrics } from '../../../../lib/details/get_metrics'; import { metricSet } from './metric_set_overview'; import { handleError } from '../../../../lib/errors'; -import { INDEX_PATTERN_ENTERPRISE_SEARCH } from '../../../../../common/constants'; import { getStats } from '../../../../lib/enterprise_search'; export function entSearchOverviewRoute(server) { @@ -34,16 +32,10 @@ export function entSearchOverviewRoute(server) { async handler(req) { const clusterUuid = req.params.clusterUuid; - const entSearchIndexPattern = prefixIndexPattern( - server.config(), - INDEX_PATTERN_ENTERPRISE_SEARCH, - req.payload.ccs - ); - try { const [stats, metrics] = await Promise.all([ - getStats(req, entSearchIndexPattern, clusterUuid), - getMetrics(req, entSearchIndexPattern, metricSet, [], { + getStats(req, clusterUuid), + getMetrics(req, 'enterprisesearch', metricSet, [], { skipClusterUuidFilter: true, }), ]); diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/_get_kibana_cluster_status.js b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/_get_kibana_cluster_status.js index 6a54e19d3493b0..373ab2cbde9b06 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/_get_kibana_cluster_status.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/_get_kibana_cluster_status.js @@ -8,9 +8,7 @@ import { get } from 'lodash'; import { getKibanasForClusters } from '../../../../lib/kibana/get_kibanas_for_clusters'; -export const getKibanaClusterStatus = (req, kbnIndexPattern, { clusterUuid }) => { +export const getKibanaClusterStatus = (req, { clusterUuid }) => { const clusters = [{ cluster_uuid: clusterUuid }]; - return getKibanasForClusters(req, kbnIndexPattern, clusters).then((kibanas) => - get(kibanas, '[0].stats') - ); + return getKibanasForClusters(req, clusters).then((kibanas) => get(kibanas, '[0].stats')); }; diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instance.ts b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instance.ts index 613ca39275c2de..dfbd8e82bb29bf 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instance.ts +++ b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instance.ts @@ -12,10 +12,7 @@ import { handleError } from '../../../../lib/errors'; // @ts-ignore import { getMetrics } from '../../../../lib/details/get_metrics'; // @ts-ignore -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; -// @ts-ignore import { metricSet } from './metric_set_instance'; -import { INDEX_PATTERN_KIBANA } from '../../../../../common/constants'; import { LegacyRequest, LegacyServer } from '../../../../types'; /** @@ -44,16 +41,13 @@ export function kibanaInstanceRoute(server: LegacyServer) { }, }, async handler(req: LegacyRequest) { - const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; const kibanaUuid = req.params.kibanaUuid; - const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs); try { const [metrics, kibanaSummary] = await Promise.all([ - getMetrics(req, kbnIndexPattern, metricSet), - getKibanaInfo(req, kbnIndexPattern, { clusterUuid, kibanaUuid }), + getMetrics(req, 'kibana', metricSet), + getKibanaInfo(req, { clusterUuid, kibanaUuid }), ]); return { diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instances.js b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instances.js index f9b3498cd684eb..f1e872d6436f28 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instances.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/instances.js @@ -6,11 +6,9 @@ */ import { schema } from '@kbn/config-schema'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { getKibanaClusterStatus } from './_get_kibana_cluster_status'; import { getKibanas } from '../../../../lib/kibana/get_kibanas'; import { handleError } from '../../../../lib/errors'; -import { INDEX_PATTERN_KIBANA } from '../../../../../common/constants'; export function kibanaInstancesRoute(server) { /** @@ -34,15 +32,12 @@ export function kibanaInstancesRoute(server) { }, }, async handler(req) { - const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; - const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs); try { const [clusterStatus, kibanas] = await Promise.all([ - getKibanaClusterStatus(req, kbnIndexPattern, { clusterUuid }), - getKibanas(req, kbnIndexPattern, { clusterUuid }), + getKibanaClusterStatus(req, { clusterUuid }), + getKibanas(req, { clusterUuid }), ]); return { diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/overview.js b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/overview.js index f9a9443c3533b6..8f77dea99868ab 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/kibana/overview.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/kibana/overview.js @@ -6,12 +6,10 @@ */ import { schema } from '@kbn/config-schema'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { getKibanaClusterStatus } from './_get_kibana_cluster_status'; import { getMetrics } from '../../../../lib/details/get_metrics'; import { metricSet } from './metric_set_overview'; import { handleError } from '../../../../lib/errors'; -import { INDEX_PATTERN_KIBANA } from '../../../../../common/constants'; export function kibanaOverviewRoute(server) { /** @@ -35,20 +33,20 @@ export function kibanaOverviewRoute(server) { }, }, async handler(req) { - const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; - const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs); try { + const moduleType = 'kibana'; + const dsDataset = 'stats'; const [clusterStatus, metrics] = await Promise.all([ - getKibanaClusterStatus(req, kbnIndexPattern, { clusterUuid }), - getMetrics(req, kbnIndexPattern, metricSet, [ + getKibanaClusterStatus(req, { clusterUuid }), + getMetrics(req, moduleType, metricSet, [ { bool: { should: [ + { term: { 'data_stream.dataset': `${moduleType}.${dsDataset}` } }, + { term: { 'metricset.name': dsDataset } }, { term: { type: 'kibana_stats' } }, - { term: { 'metricset.name': 'stats' } }, ], }, }, diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/node.js b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/node.js index 6c0ec3f0af374c..9c7749bf74903a 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/node.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/node.js @@ -9,9 +9,7 @@ import { schema } from '@kbn/config-schema'; import { getNodeInfo } from '../../../../lib/logstash/get_node_info'; import { handleError } from '../../../../lib/errors'; import { getMetrics } from '../../../../lib/details/get_metrics'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { metricSets } from './metric_set_node'; -import { INDEX_PATTERN_LOGSTASH } from '../../../../../common/constants'; const { advanced: metricSetAdvanced, overview: metricSetOverview } = metricSets; @@ -50,9 +48,7 @@ export function logstashNodeRoute(server) { }, async handler(req) { const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; - const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs); const logstashUuid = req.params.logstashUuid; let metricSet; @@ -71,18 +67,21 @@ export function logstashNodeRoute(server) { } try { + const moduleType = 'logstash'; + const dsDataset = 'node_stats'; const [metrics, nodeSummary] = await Promise.all([ - getMetrics(req, lsIndexPattern, metricSet, [ + getMetrics(req, 'logstash', metricSet, [ { bool: { should: [ + { term: { 'data_stream.dataset': `${moduleType}.${dsDataset}` } }, + { term: { 'metricset.name': dsDataset } }, { term: { type: 'logstash_stats' } }, - { term: { 'metricset.name': 'node_stats' } }, ], }, }, ]), - getNodeInfo(req, lsIndexPattern, { clusterUuid, logstashUuid }), + getNodeInfo(req, { clusterUuid, logstashUuid }), ]); return { diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/nodes.js b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/nodes.js index 051fb7d38fd417..cca02519aa7c42 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/nodes.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/nodes.js @@ -51,7 +51,7 @@ export function logstashNodesRoute(server) { try { const [clusterStatus, nodes] = await Promise.all([ - getClusterStatus(req, lsIndexPattern, { clusterUuid }), + getClusterStatus(req, { clusterUuid }), getNodes(req, lsIndexPattern, { clusterUuid }), ]); diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/overview.js b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/overview.js index 7ecd09a9d993ea..81d64e8fcdc2b9 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/overview.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/overview.js @@ -9,9 +9,7 @@ import { schema } from '@kbn/config-schema'; import { getClusterStatus } from '../../../../lib/logstash/get_cluster_status'; import { getMetrics } from '../../../../lib/details/get_metrics'; import { handleError } from '../../../../lib/errors'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; import { metricSet } from './metric_set_overview'; -import { INDEX_PATTERN_LOGSTASH } from '../../../../../common/constants'; /* * Logstash Overview route. @@ -45,24 +43,24 @@ export function logstashOverviewRoute(server) { }, }, async handler(req) { - const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; - const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs); try { + const moduleType = 'logstash'; + const dsDataset = 'node_stats'; const [metrics, clusterStatus] = await Promise.all([ - getMetrics(req, lsIndexPattern, metricSet, [ + getMetrics(req, moduleType, metricSet, [ { bool: { should: [ + { term: { 'data_stream.dataset': `${moduleType}.${dsDataset}` } }, + { term: { 'metricset.name': dsDataset } }, { term: { type: 'logstash_stats' } }, - { term: { 'metricset.name': 'node_stats' } }, ], }, }, ]), - getClusterStatus(req, lsIndexPattern, { clusterUuid }), + getClusterStatus(req, { clusterUuid }), ]); return { diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipeline.js b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipeline.js index 6b81059f0c2564..59f91b82ca4973 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipeline.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipeline.js @@ -10,8 +10,6 @@ import { handleError } from '../../../../lib/errors'; import { getPipelineVersions } from '../../../../lib/logstash/get_pipeline_versions'; import { getPipeline } from '../../../../lib/logstash/get_pipeline'; import { getPipelineVertex } from '../../../../lib/logstash/get_pipeline_vertex'; -import { prefixIndexPattern } from '../../../../../common/ccs_utils'; -import { INDEX_PATTERN_LOGSTASH } from '../../../../../common/constants'; function getPipelineVersion(versions, pipelineHash) { return pipelineHash ? versions.find(({ hash }) => hash === pipelineHash) : versions[0]; @@ -48,10 +46,8 @@ export function logstashPipelineRoute(server) { }, handler: async (req) => { const config = server.config(); - const ccs = req.payload.ccs; const clusterUuid = req.params.clusterUuid; const detailVertexId = req.payload.detailVertexId; - const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs); const pipelineId = req.params.pipelineId; // Optional params default to empty string, set to null to be more explicit. @@ -62,8 +58,6 @@ export function logstashPipelineRoute(server) { try { versions = await getPipelineVersions({ req, - config, - lsIndexPattern, clusterUuid, pipelineId, }); @@ -72,18 +66,10 @@ export function logstashPipelineRoute(server) { } const version = getPipelineVersion(versions, pipelineHash); - const promises = [getPipeline(req, config, lsIndexPattern, clusterUuid, pipelineId, version)]; + const promises = [getPipeline(req, config, clusterUuid, pipelineId, version)]; if (detailVertexId) { promises.push( - getPipelineVertex( - req, - config, - lsIndexPattern, - clusterUuid, - pipelineId, - version, - detailVertexId - ) + getPipelineVertex(req, config, clusterUuid, pipelineId, version, detailVertexId) ); } diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js index b7d86e86e7a074..ace5661b9bf988 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js @@ -8,8 +8,6 @@ import { schema } from '@kbn/config-schema'; import { getClusterStatus } from '../../../../../lib/logstash/get_cluster_status'; import { handleError } from '../../../../../lib/errors'; -import { prefixIndexPattern } from '../../../../../../common/ccs_utils'; -import { INDEX_PATTERN_LOGSTASH } from '../../../../../../common/constants'; import { getPaginatedPipelines } from '../../../../../lib/logstash/get_paginated_pipelines'; /** @@ -45,10 +43,8 @@ export function logstashClusterPipelinesRoute(server) { }, }, handler: async (req) => { - const config = server.config(); - const { ccs, pagination, sort, queryText } = req.payload; + const { pagination, sort, queryText } = req.payload; const clusterUuid = req.params.clusterUuid; - const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs); const throughputMetric = 'logstash_cluster_pipeline_throughput'; const nodesCountMetric = 'logstash_cluster_pipeline_nodes_count'; @@ -64,7 +60,6 @@ export function logstashClusterPipelinesRoute(server) { try { const response = await getPaginatedPipelines({ req, - lsIndexPattern, clusterUuid, metrics: { throughputMetric, nodesCountMetric }, pagination, @@ -74,7 +69,7 @@ export function logstashClusterPipelinesRoute(server) { return { ...response, - clusterStatus: await getClusterStatus(req, lsIndexPattern, { clusterUuid }), + clusterStatus: await getClusterStatus(req, { clusterUuid }), }; } catch (err) { throw handleError(err, req); diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js index f31e88b5b8b081..c232da925e74cc 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js @@ -8,8 +8,6 @@ import { schema } from '@kbn/config-schema'; import { getNodeInfo } from '../../../../../lib/logstash/get_node_info'; import { handleError } from '../../../../../lib/errors'; -import { prefixIndexPattern } from '../../../../../../common/ccs_utils'; -import { INDEX_PATTERN_LOGSTASH } from '../../../../../../common/constants'; import { getPaginatedPipelines } from '../../../../../lib/logstash/get_paginated_pipelines'; /** @@ -46,10 +44,8 @@ export function logstashNodePipelinesRoute(server) { }, }, handler: async (req) => { - const config = server.config(); - const { ccs, pagination, sort, queryText } = req.payload; + const { pagination, sort, queryText } = req.payload; const { clusterUuid, logstashUuid } = req.params; - const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs); const throughputMetric = 'logstash_node_pipeline_throughput'; const nodesCountMetric = 'logstash_node_pipeline_nodes_count'; @@ -66,7 +62,6 @@ export function logstashNodePipelinesRoute(server) { try { const response = await getPaginatedPipelines({ req, - lsIndexPattern, clusterUuid, logstashUuid, metrics: { throughputMetric, nodesCountMetric }, @@ -77,7 +72,7 @@ export function logstashNodePipelinesRoute(server) { return { ...response, - nodeSummary: await getNodeInfo(req, lsIndexPattern, { clusterUuid, logstashUuid }), + nodeSummary: await getNodeInfo(req, { clusterUuid, logstashUuid }), }; } catch (err) { throw handleError(err, req); diff --git a/x-pack/test/api_integration/apis/monitoring/apm/index.js b/x-pack/test/api_integration/apis/monitoring/apm/index.js index ae0623054dc6bf..b56a26d71a83aa 100644 --- a/x-pack/test/api_integration/apis/monitoring/apm/index.js +++ b/x-pack/test/api_integration/apis/monitoring/apm/index.js @@ -8,10 +8,10 @@ export default function ({ loadTestFile }) { describe('APM', () => { loadTestFile(require.resolve('./overview')); - loadTestFile(require.resolve('./overview_mb')); + // loadTestFile(require.resolve('./overview_mb')); loadTestFile(require.resolve('./instances')); - loadTestFile(require.resolve('./instances_mb')); + // loadTestFile(require.resolve('./instances_mb')); loadTestFile(require.resolve('./instance')); - loadTestFile(require.resolve('./instance_mb')); + // loadTestFile(require.resolve('./instance_mb')); }); } diff --git a/x-pack/test/api_integration/apis/monitoring/beats/index.js b/x-pack/test/api_integration/apis/monitoring/beats/index.js index 8fe09e457f685c..235afdc7fe6b21 100644 --- a/x-pack/test/api_integration/apis/monitoring/beats/index.js +++ b/x-pack/test/api_integration/apis/monitoring/beats/index.js @@ -8,10 +8,10 @@ export default function ({ loadTestFile }) { describe('Beats', () => { loadTestFile(require.resolve('./overview')); - loadTestFile(require.resolve('./overview_mb')); + // loadTestFile(require.resolve('./overview_mb')); loadTestFile(require.resolve('./list')); - loadTestFile(require.resolve('./list_mb')); + // loadTestFile(require.resolve('./list_mb')); loadTestFile(require.resolve('./detail')); - loadTestFile(require.resolve('./detail_mb')); + // loadTestFile(require.resolve('./detail_mb')); }); } diff --git a/x-pack/test/api_integration/apis/monitoring/cluster/index.js b/x-pack/test/api_integration/apis/monitoring/cluster/index.js index fb99f5e0b96056..59af14b049aa7a 100644 --- a/x-pack/test/api_integration/apis/monitoring/cluster/index.js +++ b/x-pack/test/api_integration/apis/monitoring/cluster/index.js @@ -8,8 +8,8 @@ export default function ({ loadTestFile }) { describe('cluster', () => { loadTestFile(require.resolve('./list')); - loadTestFile(require.resolve('./list_mb')); + // loadTestFile(require.resolve('./list_mb')); loadTestFile(require.resolve('./overview')); - loadTestFile(require.resolve('./overview_mb')); + // loadTestFile(require.resolve('./overview_mb')); }); } diff --git a/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js b/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js index ad864979a15fe7..167da3e360c166 100644 --- a/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js +++ b/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js @@ -8,20 +8,20 @@ export default function ({ loadTestFile }) { describe('Elasticsearch', () => { loadTestFile(require.resolve('./overview')); - loadTestFile(require.resolve('./overview_mb')); + // loadTestFile(require.resolve('./overview_mb')); loadTestFile(require.resolve('./nodes')); - loadTestFile(require.resolve('./nodes_mb')); + // loadTestFile(require.resolve('./nodes_mb')); loadTestFile(require.resolve('./node_detail')); - loadTestFile(require.resolve('./node_detail_mb')); + // loadTestFile(require.resolve('./node_detail_mb')); loadTestFile(require.resolve('./node_detail_advanced')); - loadTestFile(require.resolve('./node_detail_advanced_mb')); + // loadTestFile(require.resolve('./node_detail_advanced_mb')); loadTestFile(require.resolve('./indices')); - loadTestFile(require.resolve('./indices_mb')); + // loadTestFile(require.resolve('./indices_mb')); loadTestFile(require.resolve('./index_detail')); - loadTestFile(require.resolve('./index_detail_mb')); + // loadTestFile(require.resolve('./index_detail_mb')); loadTestFile(require.resolve('./ccr')); - loadTestFile(require.resolve('./ccr_mb')); + // loadTestFile(require.resolve('./ccr_mb')); loadTestFile(require.resolve('./ccr_shard')); - loadTestFile(require.resolve('./ccr_shard_mb')); + // loadTestFile(require.resolve('./ccr_shard_mb')); }); } diff --git a/x-pack/test/api_integration/apis/monitoring/kibana/index.js b/x-pack/test/api_integration/apis/monitoring/kibana/index.js index b54b09102bc1b2..f953104d915e8d 100644 --- a/x-pack/test/api_integration/apis/monitoring/kibana/index.js +++ b/x-pack/test/api_integration/apis/monitoring/kibana/index.js @@ -8,10 +8,10 @@ export default function ({ loadTestFile }) { describe('Kibana', () => { loadTestFile(require.resolve('./overview')); - loadTestFile(require.resolve('./overview_mb')); + // loadTestFile(require.resolve('./overview_mb')); loadTestFile(require.resolve('./listing')); - loadTestFile(require.resolve('./listing_mb')); + // loadTestFile(require.resolve('./listing_mb')); loadTestFile(require.resolve('./instance')); - loadTestFile(require.resolve('./instance_mb')); + // loadTestFile(require.resolve('./instance_mb')); }); } diff --git a/x-pack/test/api_integration/apis/monitoring/logstash/index.js b/x-pack/test/api_integration/apis/monitoring/logstash/index.js index 0caf7e360ef3a8..1a8baacacf066a 100644 --- a/x-pack/test/api_integration/apis/monitoring/logstash/index.js +++ b/x-pack/test/api_integration/apis/monitoring/logstash/index.js @@ -8,14 +8,14 @@ export default function ({ loadTestFile }) { describe('Logstash', () => { loadTestFile(require.resolve('./overview')); - loadTestFile(require.resolve('./overview_mb')); + // loadTestFile(require.resolve('./overview_mb')); loadTestFile(require.resolve('./nodes')); - loadTestFile(require.resolve('./nodes_mb')); + // loadTestFile(require.resolve('./nodes_mb')); loadTestFile(require.resolve('./node_detail')); - loadTestFile(require.resolve('./node_detail_mb')); + // loadTestFile(require.resolve('./node_detail_mb')); loadTestFile(require.resolve('./multicluster_pipelines')); - loadTestFile(require.resolve('./multicluster_pipelines_mb')); + // loadTestFile(require.resolve('./multicluster_pipelines_mb')); loadTestFile(require.resolve('./pipelines')); - loadTestFile(require.resolve('./pipelines_mb')); + // loadTestFile(require.resolve('./pipelines_mb')); }); } diff --git a/x-pack/test/api_integration/apis/monitoring/setup/collection/index.js b/x-pack/test/api_integration/apis/monitoring/setup/collection/index.js index 4690b6d6371d89..1898aedfcab0d7 100644 --- a/x-pack/test/api_integration/apis/monitoring/setup/collection/index.js +++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/index.js @@ -8,13 +8,13 @@ export default function ({ loadTestFile }) { describe('Collection', () => { loadTestFile(require.resolve('./kibana')); - loadTestFile(require.resolve('./kibana_mb')); + // loadTestFile(require.resolve('./kibana_mb')); loadTestFile(require.resolve('./kibana_exclusive')); - loadTestFile(require.resolve('./kibana_exclusive_mb')); + // loadTestFile(require.resolve('./kibana_exclusive_mb')); loadTestFile(require.resolve('./es_and_kibana')); - loadTestFile(require.resolve('./es_and_kibana_mb')); + // loadTestFile(require.resolve('./es_and_kibana_mb')); loadTestFile(require.resolve('./es_and_kibana_exclusive')); - loadTestFile(require.resolve('./es_and_kibana_exclusive_mb')); + // loadTestFile(require.resolve('./es_and_kibana_exclusive_mb')); loadTestFile(require.resolve('./detect_beats')); loadTestFile(require.resolve('./detect_beats_management')); loadTestFile(require.resolve('./detect_logstash')); diff --git a/x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum_mb/data.json.gz b/x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum_mb/data.json.gz index 163ed932c54135..d82f05dd3eb430 100644 Binary files a/x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum_mb/data.json.gz and b/x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum_mb/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb/data.json.gz b/x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb/data.json.gz index e2f6380f1010a2..4f66d024a65b96 100644 Binary files a/x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb/data.json.gz and b/x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb/data.json.gz differ