diff --git a/fleet_packages.json b/fleet_packages.json index 836e65d0e95aa..bfde442c20d22 100644 --- a/fleet_packages.json +++ b/fleet_packages.json @@ -56,6 +56,6 @@ }, { "name": "security_detection_engine", - "version": "8.12.3" + "version": "8.12.4" } ] \ No newline at end of file diff --git a/packages/kbn-search-connectors/types/native_connectors.ts b/packages/kbn-search-connectors/types/native_connectors.ts index 62ac7445e8b06..849e669c4a810 100644 --- a/packages/kbn-search-connectors/types/native_connectors.ts +++ b/packages/kbn-search-connectors/types/native_connectors.ts @@ -1793,7 +1793,7 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record @@ -345,7 +347,11 @@ export const EditorFooter = memo(function EditorFooter({ justifyContent="spaceBetween" > - {isSpaceReduced + {allowQueryCancellation && isLoading + ? i18n.translate('textBasedEditor.query.textBasedLanguagesEditor.cancel', { + defaultMessage: 'Cancel', + }) + : isSpaceReduced ? i18n.translate('textBasedEditor.query.textBasedLanguagesEditor.run', { defaultMessage: 'Run', }) @@ -361,7 +367,7 @@ export const EditorFooter = memo(function EditorFooter({ size="xs" css={css` border: 1px solid - ${Boolean(disableSubmitAction) + ${Boolean(disableSubmitAction && !allowQueryCancellation) ? euiTheme.colors.disabled : euiTheme.colors.emptyShade}; padding: 0 ${euiTheme.size.xs}; @@ -370,7 +376,7 @@ export const EditorFooter = memo(function EditorFooter({ border-radius: ${euiTheme.size.xs}; `} > - {COMMAND_KEY}⏎ + {allowQueryCancellation && isLoading ? 'X' : `${COMMAND_KEY}⏎`} diff --git a/packages/kbn-text-based-editor/src/fetch_fields_from_esql.ts b/packages/kbn-text-based-editor/src/fetch_fields_from_esql.ts index e1ef2a5d4a8b3..f1013f1a4b329 100644 --- a/packages/kbn-text-based-editor/src/fetch_fields_from_esql.ts +++ b/packages/kbn-text-based-editor/src/fetch_fields_from_esql.ts @@ -24,6 +24,7 @@ export function fetchFieldsFromESQL( query: Query | AggregateQuery, expressions: ExpressionsStart, time?: TimeRange, + abortController?: AbortController, dataView?: DataView ) { return textBasedQueryStateToAstWithValidation({ @@ -33,7 +34,15 @@ export function fetchFieldsFromESQL( }) .then((ast) => { if (ast) { - const execution = expressions.run(ast, null); + const executionContract = expressions.execute(ast, null); + + if (abortController) { + abortController.signal.onabort = () => { + executionContract.cancel(); + }; + } + + const execution = executionContract.getData(); let finalData: Datatable; let error: string | undefined; execution.pipe(pluck('result')).subscribe((resp) => { diff --git a/packages/kbn-text-based-editor/src/helpers.ts b/packages/kbn-text-based-editor/src/helpers.ts index 88df8ef6a75e4..dffddc9514449 100644 --- a/packages/kbn-text-based-editor/src/helpers.ts +++ b/packages/kbn-text-based-editor/src/helpers.ts @@ -116,6 +116,17 @@ export const parseErrors = (errors: Error[], code: string): MonacoMessage[] => { endLineNumber: Number(lineNumber), severity: monaco.MarkerSeverity.Error, }; + } else if (error.message.includes('expression was aborted')) { + return { + message: i18n.translate('textBasedEditor.query.textBasedLanguagesEditor.aborted', { + defaultMessage: 'Request was aborted', + }), + startColumn: 1, + startLineNumber: 1, + endColumn: 10, + endLineNumber: 1, + severity: monaco.MarkerSeverity.Warning, + }; } else { // unknown error message return { diff --git a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx index 7bdfce427bc21..241679734e248 100644 --- a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx +++ b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx @@ -71,7 +71,10 @@ export interface TextBasedLanguagesEditorProps { /** Callback running everytime the query changes */ onTextLangQueryChange: (query: AggregateQuery) => void; /** Callback running when the user submits the query */ - onTextLangQuerySubmit: (query?: AggregateQuery) => void; + onTextLangQuerySubmit: ( + query?: AggregateQuery, + abortController?: AbortController + ) => Promise; /** Can be used to expand/minimize the editor */ expandCodeEditor: (status: boolean) => void; /** If it is true, the editor initializes with height EDITOR_INITIAL_HEIGHT_EXPANDED */ @@ -105,6 +108,9 @@ export interface TextBasedLanguagesEditorProps { editorIsInline?: boolean; /** Disables the submit query action*/ disableSubmitAction?: boolean; + + /** when set to true enables query cancellation **/ + allowQueryCancellation?: boolean; } interface TextBasedEditorDeps { @@ -158,6 +164,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ editorIsInline, disableSubmitAction, dataTestSubj, + allowQueryCancellation, }: TextBasedLanguagesEditorProps) { const { euiTheme } = useEuiTheme(); const language = getAggregateQueryMode(query); @@ -176,7 +183,8 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ const [showLineNumbers, setShowLineNumbers] = useState(isCodeEditorExpanded); const [isCompactFocused, setIsCompactFocused] = useState(isCodeEditorExpanded); const [isCodeEditorExpandedFocused, setIsCodeEditorExpandedFocused] = useState(false); - + const [isQueryLoading, setIsQueryLoading] = useState(true); + const [abortController, setAbortController] = useState(new AbortController()); const [editorMessages, setEditorMessages] = useState<{ errors: MonacoMessage[]; warnings: MonacoMessage[]; @@ -186,12 +194,25 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ }); const onQuerySubmit = useCallback(() => { - const currentValue = editor1.current?.getValue(); - if (currentValue != null) { - setCodeStateOnSubmission(currentValue); + if (isQueryLoading && allowQueryCancellation) { + abortController?.abort(); + setIsQueryLoading(false); + } else { + setIsQueryLoading(true); + const abc = new AbortController(); + setAbortController(abc); + + const currentValue = editor1.current?.getValue(); + if (currentValue != null) { + setCodeStateOnSubmission(currentValue); + } + onTextLangQuerySubmit({ [language]: currentValue } as AggregateQuery, abc); } - onTextLangQuerySubmit({ [language]: currentValue } as AggregateQuery); - }, [language, onTextLangQuerySubmit]); + }, [language, onTextLangQuerySubmit, abortController, isQueryLoading, allowQueryCancellation]); + + useEffect(() => { + if (!isLoading) setIsQueryLoading(false); + }, [isLoading]); const [documentationSections, setDocumentationSections] = useState(); @@ -311,12 +332,13 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ const { cache: esqlFieldsCache, memoizedFieldsFromESQL } = useMemo(() => { // need to store the timing of the first request so we can atomically clear the cache per query const fn = memoize( - (...args: [{ esql: string }, ExpressionsStart]) => ({ + (...args: [{ esql: string }, ExpressionsStart, undefined, AbortController?]) => ({ timestamp: Date.now(), result: fetchFieldsFromESQL(...args), }), ({ esql }) => esql ); + return { cache: fn.cache, memoizedFieldsFromESQL: fn }; }, []); @@ -334,7 +356,12 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ // Check if there's a stale entry and clear it clearCacheWhenOld(esqlFieldsCache, esqlQuery.esql); try { - const table = await memoizedFieldsFromESQL(esqlQuery, expressions).result; + const table = await memoizedFieldsFromESQL( + esqlQuery, + expressions, + undefined, + abortController + ).result; return table?.columns.map((c) => ({ name: c.name, type: c.meta.type })) || []; } catch (e) { // no action yet @@ -352,7 +379,14 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ return policies.map(({ type, query: policyQuery, ...rest }) => rest); }, }), - [dataViews, expressions, indexManagementApiService, esqlFieldsCache, memoizedFieldsFromESQL] + [ + dataViews, + expressions, + indexManagementApiService, + esqlFieldsCache, + memoizedFieldsFromESQL, + abortController, + ] ); const queryValidation = useCallback( @@ -867,7 +901,8 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ disableSubmitAction={disableSubmitAction} hideRunQueryText={hideRunQueryText} isSpaceReduced={isSpaceReduced} - isLoading={isLoading} + isLoading={isQueryLoading} + allowQueryCancellation={allowQueryCancellation} /> )} @@ -954,13 +989,16 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ lines={lines} containerCSS={styles.bottomContainer} onErrorClick={onErrorClick} - runQuery={onQuerySubmit} + runQuery={() => { + onQuerySubmit(); + }} detectTimestamp={detectTimestamp} hideRunQueryText={hideRunQueryText} editorIsInline={editorIsInline} disableSubmitAction={disableSubmitAction} isSpaceReduced={isSpaceReduced} - isLoading={isLoading} + isLoading={isQueryLoading} + allowQueryCancellation={allowQueryCancellation} {...editorMessages} /> )} diff --git a/src/plugins/unified_search/public/query_string_input/query_bar_top_row.tsx b/src/plugins/unified_search/public/query_string_input/query_bar_top_row.tsx index 0e4020f5c70fa..dd9fb37258ed3 100644 --- a/src/plugins/unified_search/public/query_string_input/query_bar_top_row.tsx +++ b/src/plugins/unified_search/public/query_string_input/query_bar_top_row.tsx @@ -717,7 +717,7 @@ export const QueryBarTopRow = React.memo( errors={props.textBasedLanguageModeErrors} warning={props.textBasedLanguageModeWarning} detectTimestamp={detectTimestamp} - onTextLangQuerySubmit={() => + onTextLangQuerySubmit={async () => onSubmit({ query: queryRef.current, dateRange: dateRangeRef.current, diff --git a/x-pack/packages/ml/anomaly_utils/anomaly_utils.ts b/x-pack/packages/ml/anomaly_utils/anomaly_utils.ts index 4865aed1e4e97..8ef9fee14a273 100644 --- a/x-pack/packages/ml/anomaly_utils/anomaly_utils.ts +++ b/x-pack/packages/ml/anomaly_utils/anomaly_utils.ts @@ -59,6 +59,10 @@ export interface MlEntityField { * Optional entity field operation */ operation?: MlEntityFieldOperation; + /** + * Optional cardinality of field + */ + cardinality?: number; } // List of function descriptions for which actual values from record level results should be displayed. diff --git a/x-pack/plugins/apm/ftr_e2e/cypress/e2e/service_inventory/service_inventory.cy.ts b/x-pack/plugins/apm/ftr_e2e/cypress/e2e/service_inventory/service_inventory.cy.ts index c0c3c032a0e61..5c50e79c145aa 100644 --- a/x-pack/plugins/apm/ftr_e2e/cypress/e2e/service_inventory/service_inventory.cy.ts +++ b/x-pack/plugins/apm/ftr_e2e/cypress/e2e/service_inventory/service_inventory.cy.ts @@ -115,7 +115,8 @@ describe('Service inventory', () => { }); }); - describe('Table search', () => { + // Skipping this until we enable the table search on the Service inventory view + describe.skip('Table search', () => { beforeEach(() => { cy.updateAdvancedSettings({ 'observability:apmEnableTableSearchBar': true, diff --git a/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx b/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx index 08e7a840b5dfb..ba55defaaf4d7 100644 --- a/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx @@ -357,6 +357,7 @@ export function ServiceList({ const tableSearchBar: TableSearchBar = useMemo(() => { return { + isEnabled: false, fieldsToSearch: ['serviceName'], maxCountExceeded, onChangeSearchQuery, diff --git a/x-pack/plugins/apm/public/components/shared/managed_table/index.tsx b/x-pack/plugins/apm/public/components/shared/managed_table/index.tsx index 7d6307d32ffb8..ae14f63f8d72b 100644 --- a/x-pack/plugins/apm/public/components/shared/managed_table/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/managed_table/index.tsx @@ -111,7 +111,7 @@ function UnoptimizedManagedTable(props: { const { core } = useApmPluginContext(); const isTableSearchBarEnabled = core.uiSettings.get( apmEnableTableSearchBar, - false + true ); const { diff --git a/x-pack/plugins/cases/public/components/all_cases/severity_filter.test.tsx b/x-pack/plugins/cases/public/components/all_cases/severity_filter.test.tsx index 6924cbd13f1c7..28be7c63f22b3 100644 --- a/x-pack/plugins/cases/public/components/all_cases/severity_filter.test.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/severity_filter.test.tsx @@ -14,7 +14,9 @@ import { screen, waitFor } from '@testing-library/react'; import { waitForEuiPopoverOpen } from '@elastic/eui/lib/test/rtl'; import { SeverityFilter } from './severity_filter'; -describe('Severity form field', () => { +// FLAKY: https://github.com/elastic/kibana/issues/176336 +// FLAKY: https://github.com/elastic/kibana/issues/176337 +describe.skip('Severity form field', () => { const onChange = jest.fn(); let appMockRender: AppMockRenderer; const props = { diff --git a/x-pack/plugins/cases/public/components/app/routes.test.tsx b/x-pack/plugins/cases/public/components/app/routes.test.tsx index 9c435e2b163ba..91b4b1ef5227f 100644 --- a/x-pack/plugins/cases/public/components/app/routes.test.tsx +++ b/x-pack/plugins/cases/public/components/app/routes.test.tsx @@ -60,7 +60,8 @@ describe('Cases routes', () => { }); }); - describe('Case view', () => { + // FLAKY: https://github.com/elastic/kibana/issues/163263 + describe.skip('Case view', () => { it.each(getCaseViewPaths())( 'navigates to the cases view page for path: %s', async (path: string) => { @@ -84,7 +85,9 @@ describe('Cases routes', () => { ); }); - describe('Create case', () => { + // FLAKY: https://github.com/elastic/kibana/issues/175229 + // FLAKY: https://github.com/elastic/kibana/issues/175230 + describe.skip('Create case', () => { it('navigates to the create case page', () => { renderWithRouter(['/cases/create']); expect(screen.getByText('Create case')).toBeInTheDocument(); @@ -96,7 +99,9 @@ describe('Cases routes', () => { }); }); - describe('Cases settings', () => { + // FLAKY: https://github.com/elastic/kibana/issues/175231 + // FLAKY: https://github.com/elastic/kibana/issues/175232 + describe.skip('Cases settings', () => { it('navigates to the cases settings page', () => { renderWithRouter(['/cases/configure']); expect(screen.getByText('Settings')).toBeInTheDocument(); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx index 111b0a2113403..9ea517d45eea1 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx @@ -116,10 +116,10 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, > {Array.isArray(topValues) ? topValues.map((value) => { - const fieldValue = - value.key_as_string ?? (value.key ? value.key.toString() : EMPTY_EXAMPLE); + const fieldValue = value.key_as_string ?? (value.key ? value.key.toString() : ''); + const displayValue = fieldValue ?? EMPTY_EXAMPLE; return ( - + = ({ stats, fieldFormat, barColor, compressed, /> {fieldName !== undefined && - fieldValue !== undefined && + displayValue !== undefined && onAddFilter !== undefined ? (
= ({ stats, fieldFormat, barColor, compressed, 'xpack.dataVisualizer.dataGrid.field.addFilterAriaLabel', { defaultMessage: 'Filter for {fieldName}: "{value}"', - values: { fieldName, value: fieldValue }, + values: { fieldName, value: displayValue }, } )} - data-test-subj={`dvFieldDataTopValuesAddFilterButton-${fieldName}-${fieldValue}`} + data-test-subj={`dvFieldDataTopValuesAddFilterButton-${fieldName}-${displayValue}`} style={{ minHeight: 'auto', minWidth: 'auto', @@ -172,10 +172,10 @@ export const TopValues: FC = ({ stats, fieldFormat, barColor, compressed, 'xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel', { defaultMessage: 'Filter out {fieldName}: "{value}"', - values: { fieldName, value: fieldValue }, + values: { fieldName, value: displayValue }, } )} - data-test-subj={`dvFieldDataTopValuesExcludeFilterButton-${fieldName}-${fieldValue}`} + data-test-subj={`dvFieldDataTopValuesExcludeFilterButton-${fieldName}-${displayValue}`} style={{ minHeight: 'auto', minWidth: 'auto', diff --git a/x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts index 65c882ba551d9..9b85720fc1df4 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts @@ -14,9 +14,9 @@ import { mlTimefilterRefresh$, useTimefilter } from '@kbn/ml-date-picker'; import { merge } from 'rxjs'; import { RandomSampler } from '@kbn/ml-random-sampler-utils'; import { mapAndFlattenFilters } from '@kbn/data-plugin/public'; -import { Query } from '@kbn/es-query'; +import { buildEsQuery, Query } from '@kbn/es-query'; import { SearchQueryLanguage } from '@kbn/ml-query-utils'; -import { createMergedEsQuery } from '../../index_data_visualizer/utils/saved_search_utils'; +import { getEsQueryConfig } from '@kbn/data-plugin/common'; import { useDataDriftStateManagerContext } from '../../data_drift/use_state_manager'; import type { InitialSettings } from '../../data_drift/use_data_drift_result'; import { @@ -74,7 +74,7 @@ export const useData = ( () => { const searchQuery = searchString !== undefined && searchQueryLanguage !== undefined - ? { query: searchString, language: searchQueryLanguage } + ? ({ query: searchString, language: searchQueryLanguage } as Query) : undefined; const timefilterActiveBounds = timeRange ?? timefilter.getActiveBounds(); @@ -90,24 +90,24 @@ export const useData = ( runtimeFieldMap: selectedDataView.getRuntimeMappings(), }; - const refQuery = createMergedEsQuery( - searchQuery, + const refQuery = buildEsQuery( + selectedDataView, + searchQuery ?? [], mapAndFlattenFilters([ ...queryManager.filterManager.getFilters(), ...(referenceStateManager.filters ?? []), ]), - selectedDataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); - const compQuery = createMergedEsQuery( - searchQuery, + const compQuery = buildEsQuery( + selectedDataView, + searchQuery ?? [], mapAndFlattenFilters([ ...queryManager.filterManager.getFilters(), ...(comparisonStateManager.filters ?? []), ]), - selectedDataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); return { diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts b/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts index 07b74677e8ea9..05f24bdcb7b68 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts @@ -8,6 +8,7 @@ import { chunk, cloneDeep, flatten } from 'lodash'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { lastValueFrom } from 'rxjs'; +import { getEsQueryConfig } from '@kbn/data-plugin/common'; import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { @@ -30,7 +31,7 @@ import { computeChi2PValue, type Histogram } from '@kbn/ml-chi2test'; import { mapAndFlattenFilters } from '@kbn/data-plugin/public'; import type { AggregationsMultiTermsBucketKeys } from '@elastic/elasticsearch/lib/api/types'; -import { createMergedEsQuery } from '../index_data_visualizer/utils/saved_search_utils'; +import { buildEsQuery } from '@kbn/es-query'; import { useDataVisualizerKibana } from '../kibana_context'; import { useDataDriftStateManagerContext } from './use_state_manager'; @@ -758,18 +759,18 @@ export const useFetchDataComparisonResult = ( const kqlQuery = searchString !== undefined && searchQueryLanguage !== undefined - ? { query: searchString, language: searchQueryLanguage } + ? ({ query: searchString, language: searchQueryLanguage } as Query) : undefined; const refDataQuery = getDataComparisonQuery({ - searchQuery: createMergedEsQuery( - kqlQuery, + searchQuery: buildEsQuery( + currentDataView, + kqlQuery ?? [], mapAndFlattenFilters([ ...queryManager.filterManager.getFilters(), ...(referenceStateManager.filters ?? []), ]), - currentDataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ), datetimeField: currentDataView?.timeFieldName, runtimeFields, @@ -827,14 +828,14 @@ export const useFetchDataComparisonResult = ( setLoaded(0.25); const prodDataQuery = getDataComparisonQuery({ - searchQuery: createMergedEsQuery( - kqlQuery, + searchQuery: buildEsQuery( + currentDataView, + kqlQuery ?? [], mapAndFlattenFilters([ ...queryManager.filterManager.getFilters(), ...(comparisonStateManager.filters ?? []), ]), - currentDataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ), datetimeField: currentDataView?.timeFieldName, runtimeFields, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx index 0faa236e30c6f..2f91dce01b456 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx @@ -675,7 +675,7 @@ export const IndexDataVisualizerESQL: FC = (dataVi // Query that has been typed, but has not submitted with cmd + enter const [localQuery, setLocalQuery] = useState({ esql: '' }); - const onQueryUpdate = (q?: AggregateQuery) => { + const onQueryUpdate = async (q?: AggregateQuery) => { // When user submits a new query // resets all current requests and other data if (cancelOverallStatsRequest) { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index cc17387886071..5d8ebe9e44d57 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -8,6 +8,7 @@ import { css } from '@emotion/react'; import React, { FC, useEffect, useMemo, useState, useCallback, useRef } from 'react'; import type { Required } from 'utility-types'; +import { getEsQueryConfig } from '@kbn/data-plugin/common'; import { useEuiBreakpoint, @@ -21,7 +22,7 @@ import { EuiTitle, } from '@elastic/eui'; -import { type Filter, FilterStateStore, type Query } from '@kbn/es-query'; +import { type Filter, FilterStateStore, type Query, buildEsQuery } from '@kbn/es-query'; import { generateFilters } from '@kbn/data-plugin/public'; import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; import { usePageUrlState, useUrlState } from '@kbn/ml-url-state'; @@ -62,7 +63,6 @@ import { DocumentCountContent } from '../../../common/components/document_count_ import { OMIT_FIELDS } from '../../../../../common/constants'; import { SearchPanel } from '../search_panel'; import { ActionsPanel } from '../actions_panel'; -import { createMergedEsQuery } from '../../utils/saved_search_utils'; import { DataVisualizerDataViewManagement } from '../data_view_management'; import type { GetAdditionalLinks } from '../../../common/components/results_links'; import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; @@ -389,14 +389,14 @@ export const IndexDataVisualizerView: FC = (dataVi language: searchQueryLanguage, }; - const combinedQuery = createMergedEsQuery( + const combinedQuery = buildEsQuery( + currentDataView, { query: searchString || '', language: searchQueryLanguage, }, data.query.filterManager.getFilters() ?? [], - currentDataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); setSearchParams({ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx index 3ad691bbe11ce..d0f6812c4e253 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx @@ -5,13 +5,13 @@ * 2.0. */ -import type { Filter, Query, TimeRange } from '@kbn/es-query'; +import { buildEsQuery, Filter, Query, TimeRange } from '@kbn/es-query'; import { i18n } from '@kbn/i18n'; import React, { useEffect, useState } from 'react'; import { isDefined } from '@kbn/ml-is-defined'; import { DataView } from '@kbn/data-views-plugin/common'; import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; -import { createMergedEsQuery } from '../../utils/saved_search_utils'; +import { getEsQueryConfig } from '@kbn/data-plugin/common'; import { useDataVisualizerKibana } from '../../../kibana_context'; export const SearchPanelContent = ({ @@ -63,16 +63,17 @@ export const SearchPanelContent = ({ const searchHandler = ({ query, filters }: { query?: Query; filters?: Filter[] }) => { const mergedQuery = isDefined(query) ? query : searchInput; const mergedFilters = isDefined(filters) ? filters : queryManager.filterManager.getFilters(); + try { if (mergedFilters) { queryManager.filterManager.setFilters(mergedFilters); } - const combinedQuery = createMergedEsQuery( - mergedQuery, - queryManager.filterManager.getFilters() ?? [], + const combinedQuery = buildEsQuery( dataView, - uiSettings + mergedQuery ? [mergedQuery] : [], + queryManager.filterManager.getFilters() ?? [], + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); setSearchParams({ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index b012d049ae04f..4570a2019af26 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -137,10 +137,11 @@ export const useDataVisualizerGridData = ( }); if (searchData === undefined || dataVisualizerListState.searchString !== '') { - if (dataVisualizerListState.filters) { + if (filterManager) { const globalFilters = filterManager?.getGlobalFilters(); - if (filterManager) filterManager.setFilters(dataVisualizerListState.filters); + if (dataVisualizerListState.filters) + filterManager.setFilters(dataVisualizerListState.filters); if (globalFilters) filterManager?.addFilters(globalFilters); } return { @@ -169,6 +170,7 @@ export const useDataVisualizerGridData = ( currentFilters, }), lastRefresh, + data.query.filterManager, ]); const _timeBuckets = useTimeBuckets(); diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts index c43483a34e34c..2b25a5e8d2b8c 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.test.ts @@ -5,14 +5,10 @@ * 2.0. */ -import { - getQueryFromSavedSearchObject, - createMergedEsQuery, - getEsQueryFromSavedSearch, -} from './saved_search_utils'; +import { getQueryFromSavedSearchObject, getEsQueryFromSavedSearch } from './saved_search_utils'; import type { SavedSearchSavedObject } from '../../../../common/types'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; -import { type Filter, FilterStateStore } from '@kbn/es-query'; +import { FilterStateStore } from '@kbn/es-query'; import { stubbedSavedObjectIndexPattern } from '@kbn/data-views-plugin/common/data_view.stub'; import { DataView } from '@kbn/data-views-plugin/public'; import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks'; @@ -217,80 +213,6 @@ describe('getQueryFromSavedSearchObject()', () => { }); }); -describe('createMergedEsQuery()', () => { - const luceneQuery = { - query: 'responsetime:>50', - language: 'lucene', - }; - const kqlQuery = { - query: 'responsetime > 49', - language: 'kuery', - }; - const mockFilters: Filter[] = [ - { - meta: { - index: '90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0', - negate: false, - disabled: false, - alias: null, - type: 'phrase', - key: 'airline', - params: { - query: 'ASA', - }, - }, - query: { - match: { - airline: { - query: 'ASA', - type: 'phrase', - }, - }, - }, - $state: { - store: 'appState' as FilterStateStore, - }, - }, - ]; - - it('return formatted ES bool query with both the original query and filters combined', () => { - expect(createMergedEsQuery(luceneQuery, mockFilters)).toEqual({ - bool: { - filter: [{ match_phrase: { airline: { query: 'ASA' } } }], - must: [{ query_string: { query: 'responsetime:>50' } }], - must_not: [], - should: [], - }, - }); - expect(createMergedEsQuery(kqlQuery, mockFilters)).toEqual({ - bool: { - filter: [{ match_phrase: { airline: { query: 'ASA' } } }], - minimum_should_match: 1, - must_not: [], - should: [{ range: { responsetime: { gt: '49' } } }], - }, - }); - }); - it('return formatted ES bool query without filters ', () => { - expect(createMergedEsQuery(luceneQuery)).toEqual({ - bool: { - filter: [], - must: [{ query_string: { query: 'responsetime:>50' } }], - must_not: [], - should: [], - }, - }); - expect(createMergedEsQuery(kqlQuery)).toEqual({ - bool: { - filter: [], - minimum_should_match: 1, - must_not: [], - should: [{ range: { responsetime: { gt: '49' } } }], - }, - }); - }); -}); - describe('getEsQueryFromSavedSearch()', () => { it('return undefined if saved search is not provided', () => { expect( diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index 04bc52bf08057..3ecc8a3a7a3d8 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -9,22 +9,15 @@ // `x-pack/plugins/apm/public/components/app/correlations/progress_controls.tsx` import { cloneDeep } from 'lodash'; import { IUiSettingsClient } from '@kbn/core/public'; -import { - fromKueryExpression, - toElasticsearchQuery, - buildQueryFromFilters, - buildEsQuery, - Query, - Filter, - AggregateQuery, -} from '@kbn/es-query'; +import { buildEsQuery, Query, Filter } from '@kbn/es-query'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { DataView } from '@kbn/data-views-plugin/public'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; -import { getEsQueryConfig, isQuery, SearchSource } from '@kbn/data-plugin/common'; +import { getEsQueryConfig, SearchSource } from '@kbn/data-plugin/common'; import { FilterManager, mapAndFlattenFilters } from '@kbn/data-plugin/public'; import { getDefaultDSLQuery } from '@kbn/ml-query-utils'; -import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { isDefined } from '@kbn/ml-is-defined'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; /** @@ -59,53 +52,8 @@ export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObjec return parsed; } -/** - * Create an Elasticsearch query that combines both lucene/kql query string and filters - * Should also form a valid query if only the query or filters is provided - */ -export function createMergedEsQuery( - query?: Query | AggregateQuery | undefined, - filters?: Filter[], - dataView?: DataView, - uiSettings?: IUiSettingsClient -) { - let combinedQuery = getDefaultDSLQuery() as QueryDslQueryContainer; - - if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { - const ast = fromKueryExpression(query.query); - if (query.query !== '') { - combinedQuery = toElasticsearchQuery(ast, dataView); - } - if (combinedQuery.bool !== undefined) { - const filterQuery = buildQueryFromFilters(filters, dataView); - - if (!Array.isArray(combinedQuery.bool.filter)) { - combinedQuery.bool.filter = - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; - } - - if (!Array.isArray(combinedQuery.bool.must_not)) { - combinedQuery.bool.must_not = - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; - } - - combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; - } - } else { - combinedQuery = buildEsQuery( - dataView, - query ? [query] : [], - filters ? filters : [], - uiSettings ? getEsQueryConfig(uiSettings) : undefined - ); - } - - return combinedQuery; -} - -function getSavedSearchSource(savedSearch: SavedSearch) { - return savedSearch && +function getSavedSearchSource(savedSearch?: SavedSearch | null) { + return isDefined(savedSearch) && 'searchSource' in savedSearch && savedSearch?.searchSource instanceof SearchSource ? savedSearch.searchSource @@ -131,11 +79,15 @@ export function getEsQueryFromSavedSearch({ filters?: Filter[]; filterManager?: FilterManager; }) { - if (!dataView || !savedSearch) return; + if (!dataView && !savedSearch) return; const userQuery = query; const userFilters = filters; + if (filterManager && userFilters) { + filterManager.addFilters(userFilters); + } + const savedSearchSource = getSavedSearchSource(savedSearch); // If saved search has a search source with nested parent @@ -146,8 +98,8 @@ export function getEsQueryFromSavedSearch({ // Flattened query from search source may contain a clause that narrows the time range // which might interfere with global time pickers so we need to remove const savedQuery = - cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultDSLQuery(); - const timeField = savedSearch.searchSource.getField('index')?.timeFieldName; + cloneDeep(savedSearchSource.getSearchRequestBody()?.query) ?? getDefaultDSLQuery(); + const timeField = savedSearchSource.getField('index')?.timeFieldName; if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) { savedQuery.bool.filter = savedQuery.bool.filter.filter( @@ -155,6 +107,7 @@ export function getEsQueryFromSavedSearch({ !(c.hasOwnProperty('range') && c.range?.hasOwnProperty(timeField)) ); } + return { searchQuery: savedQuery, searchString: userQuery.query, @@ -163,39 +116,38 @@ export function getEsQueryFromSavedSearch({ } // If no saved search available, use user's query and filters - if (!savedSearch && userQuery) { - if (filterManager && userFilters) filterManager.addFilters(userFilters); - - const combinedQuery = createMergedEsQuery( - userQuery, - Array.isArray(userFilters) ? userFilters : [], + if ( + !savedSearch && + (userQuery || userFilters || (filterManager && filterManager.getGlobalFilters()?.length > 0)) + ) { + const combinedQuery = buildEsQuery( dataView, - uiSettings + userQuery ?? [], + filterManager?.getFilters() ?? [], + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); return { searchQuery: combinedQuery, - searchString: userQuery.query, - queryLanguage: userQuery.language as SearchQueryLanguage, + searchString: userQuery?.query ?? '', + queryLanguage: (userQuery?.language ?? 'kuery') as SearchQueryLanguage, }; } // If saved search available, merge saved search with the latest user query or filters // which might differ from extracted saved search data if (savedSearchSource) { - const globalFilters = filterManager?.getGlobalFilters(); // FIXME: Add support for AggregateQuery type #150091 const currentQuery = userQuery ?? (savedSearchSource.getField('query') as Query); const currentFilters = userFilters ?? mapAndFlattenFilters(savedSearchSource.getField('filter') as Filter[]); - if (filterManager) filterManager.setFilters(currentFilters); - if (globalFilters) filterManager?.addFilters(globalFilters); + if (filterManager) filterManager.addFilters(currentFilters); - const combinedQuery = createMergedEsQuery( + const combinedQuery = buildEsQuery( + dataView, currentQuery, filterManager ? filterManager?.getFilters() : currentFilters, - dataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); return { diff --git a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts index 475862664c336..803fcbf169935 100644 --- a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts +++ b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts @@ -15,7 +15,11 @@ import type { LensPluginStartDependencies } from '../../../plugin'; import type { DatasourceMap, VisualizationMap } from '../../../types'; import { suggestionsApi } from '../../../lens_suggestions_api'; -export const getQueryColumns = async (query: AggregateQuery, deps: LensPluginStartDependencies) => { +export const getQueryColumns = async ( + query: AggregateQuery, + deps: LensPluginStartDependencies, + abortController?: AbortController +) => { // Fetching only columns for ES|QL for performance reasons with limit 0 // Important note: ES doesnt return the warnings for 0 limit, // I am skipping them in favor of performance now @@ -24,7 +28,12 @@ export const getQueryColumns = async (query: AggregateQuery, deps: LensPluginSta if ('esql' in performantQuery && performantQuery.esql) { performantQuery.esql = `${performantQuery.esql} | limit 0`; } - const table = await fetchFieldsFromESQL(performantQuery, deps.expressions); + const table = await fetchFieldsFromESQL( + performantQuery, + deps.expressions, + undefined, + abortController + ); return table?.columns; }; @@ -34,7 +43,8 @@ export const getSuggestions = async ( datasourceMap: DatasourceMap, visualizationMap: VisualizationMap, adHocDataViews: DataViewSpec[], - setErrors: (errors: Error[]) => void + setErrors: (errors: Error[]) => void, + abortController?: AbortController ) => { try { let indexPattern = ''; @@ -55,7 +65,7 @@ export const getSuggestions = async ( if (dataView.fields.getByName('@timestamp')?.type === 'date' && !dataViewSpec) { dataView.timeFieldName = '@timestamp'; } - const columns = await getQueryColumns(query, deps); + const columns = await getQueryColumns(query, deps, abortController); const context = { dataViewSpec: dataView?.toSpec(), fieldName: '', diff --git a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/lens_configuration_flyout.tsx b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/lens_configuration_flyout.tsx index 834929d4ca2a5..3e2bf4f60aa2b 100644 --- a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/lens_configuration_flyout.tsx +++ b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/lens_configuration_flyout.tsx @@ -279,14 +279,15 @@ export function LensEditConfigurationFlyout({ const adHocDataViews = Object.values(attributes.state.adHocDataViews ?? {}); const runQuery = useCallback( - async (q) => { + async (q, abortController) => { const attrs = await getSuggestions( q, startDependencies, datasourceMap, visualizationMap, adHocDataViews, - setErrors + setErrors, + abortController ); if (attrs) { setCurrentAttributes?.(attrs); @@ -442,13 +443,13 @@ export function LensEditConfigurationFlyout({ hideMinimizeButton editorIsInline hideRunQueryText - disableSubmitAction={isEqual(query, prevQuery.current)} - onTextLangQuerySubmit={(q) => { + onTextLangQuerySubmit={async (q, a) => { if (q) { - runQuery(q); + await runQuery(q, a); } }} isDisabled={false} + allowQueryCancellation={true} /> )} diff --git a/x-pack/plugins/maps/public/classes/layers/invalid_layer.ts b/x-pack/plugins/maps/public/classes/layers/invalid_layer.ts new file mode 100644 index 0000000000000..eed2ab37b32e0 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/layers/invalid_layer.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable max-classes-per-file */ + +import { i18n } from '@kbn/i18n'; +import { LayerDescriptor } from '../../../common/descriptor_types'; +import { AbstractLayer } from './layer'; +import { AbstractSource } from '../sources/source'; +import { IStyle } from '../styles/style'; + +class InvalidSource extends AbstractSource { + constructor(id?: string) { + super({ + id, + type: 'INVALID', + }); + } +} + +export class InvalidLayer extends AbstractLayer { + private readonly _error: Error; + private readonly _style: IStyle; + + constructor(layerDescriptor: LayerDescriptor, error: Error) { + super({ + layerDescriptor, + source: new InvalidSource(layerDescriptor.sourceDescriptor?.id), + }); + this._error = error; + this._style = { + getType() { + return 'INVALID'; + }, + renderEditor() { + return null; + }, + }; + } + + hasErrors() { + return true; + } + + getErrors() { + return [ + { + title: i18n.translate('xpack.maps.invalidLayer.errorTitle', { + defaultMessage: `Unable to create layer`, + }), + body: this._error.message, + }, + ]; + } + + getStyleForEditing() { + return this._style; + } + + getStyle() { + return this._style; + } + + getCurrentStyle() { + return this._style; + } + + getMbLayerIds() { + return []; + } + + ownsMbLayerId() { + return false; + } + + ownsMbSourceId() { + return false; + } + + syncLayerWithMB() {} + + getLayerTypeIconName() { + return 'error'; + } +} diff --git a/x-pack/plugins/maps/public/classes/layers/layer.tsx b/x-pack/plugins/maps/public/classes/layers/layer.tsx index 1fccaf7f6d0a5..aa39cf017eb0d 100644 --- a/x-pack/plugins/maps/public/classes/layers/layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/layer.tsx @@ -245,7 +245,7 @@ export class AbstractLayer implements ILayer { const sourceDisplayName = source ? await source.getDisplayName() : await this.getSource().getDisplayName(); - return sourceDisplayName || `Layer ${this._descriptor.id}`; + return sourceDisplayName || this._descriptor.id; } async getAttributions(): Promise { diff --git a/x-pack/plugins/maps/public/selectors/map_selectors.ts b/x-pack/plugins/maps/public/selectors/map_selectors.ts index ec035ac1c5623..78950c1ab2e7f 100644 --- a/x-pack/plugins/maps/public/selectors/map_selectors.ts +++ b/x-pack/plugins/maps/public/selectors/map_selectors.ts @@ -23,6 +23,7 @@ import { import { VectorStyle } from '../classes/styles/vector/vector_style'; import { isLayerGroup, LayerGroup } from '../classes/layers/layer_group'; import { HeatmapLayer } from '../classes/layers/heatmap_layer'; +import { InvalidLayer } from '../classes/layers/invalid_layer'; import { getTimeFilter } from '../kibana_services'; import { getChartsPaletteServiceGetColor } from '../reducers/non_serializable_instances'; import { copyPersistentState, TRACKED_LAYER_DESCRIPTOR } from '../reducers/copy_persistent_state'; @@ -76,54 +77,58 @@ export function createLayerInstance( customIcons: CustomIcon[], chartsPaletteServiceGetColor?: (value: string) => string | null ): ILayer { - if (layerDescriptor.type === LAYER_TYPE.LAYER_GROUP) { - return new LayerGroup({ layerDescriptor: layerDescriptor as LayerGroupDescriptor }); - } + try { + if (layerDescriptor.type === LAYER_TYPE.LAYER_GROUP) { + return new LayerGroup({ layerDescriptor: layerDescriptor as LayerGroupDescriptor }); + } - const source: ISource = createSourceInstance(layerDescriptor.sourceDescriptor); - switch (layerDescriptor.type) { - case LAYER_TYPE.RASTER_TILE: - return new RasterTileLayer({ layerDescriptor, source: source as IRasterSource }); - case LAYER_TYPE.EMS_VECTOR_TILE: - return new EmsVectorTileLayer({ - layerDescriptor: layerDescriptor as EMSVectorTileLayerDescriptor, - source: source as EMSTMSSource, - }); - case LAYER_TYPE.HEATMAP: - return new HeatmapLayer({ - layerDescriptor: layerDescriptor as HeatmapLayerDescriptor, - source: source as ESGeoGridSource, - }); - case LAYER_TYPE.GEOJSON_VECTOR: - return new GeoJsonVectorLayer({ - layerDescriptor: layerDescriptor as VectorLayerDescriptor, - source: source as IVectorSource, - joins: createJoinInstances( - layerDescriptor as VectorLayerDescriptor, - source as IVectorSource - ), - customIcons, - chartsPaletteServiceGetColor, - }); - case LAYER_TYPE.BLENDED_VECTOR: - return new BlendedVectorLayer({ - layerDescriptor: layerDescriptor as VectorLayerDescriptor, - source: source as IVectorSource, - customIcons, - chartsPaletteServiceGetColor, - }); - case LAYER_TYPE.MVT_VECTOR: - return new MvtVectorLayer({ - layerDescriptor: layerDescriptor as VectorLayerDescriptor, - source: source as IVectorSource, - joins: createJoinInstances( - layerDescriptor as VectorLayerDescriptor, - source as IVectorSource - ), - customIcons, - }); - default: - throw new Error(`Unrecognized layerType ${layerDescriptor.type}`); + const source: ISource = createSourceInstance(layerDescriptor.sourceDescriptor); + switch (layerDescriptor.type) { + case LAYER_TYPE.RASTER_TILE: + return new RasterTileLayer({ layerDescriptor, source: source as IRasterSource }); + case LAYER_TYPE.EMS_VECTOR_TILE: + return new EmsVectorTileLayer({ + layerDescriptor: layerDescriptor as EMSVectorTileLayerDescriptor, + source: source as EMSTMSSource, + }); + case LAYER_TYPE.HEATMAP: + return new HeatmapLayer({ + layerDescriptor: layerDescriptor as HeatmapLayerDescriptor, + source: source as ESGeoGridSource, + }); + case LAYER_TYPE.GEOJSON_VECTOR: + return new GeoJsonVectorLayer({ + layerDescriptor: layerDescriptor as VectorLayerDescriptor, + source: source as IVectorSource, + joins: createJoinInstances( + layerDescriptor as VectorLayerDescriptor, + source as IVectorSource + ), + customIcons, + chartsPaletteServiceGetColor, + }); + case LAYER_TYPE.BLENDED_VECTOR: + return new BlendedVectorLayer({ + layerDescriptor: layerDescriptor as VectorLayerDescriptor, + source: source as IVectorSource, + customIcons, + chartsPaletteServiceGetColor, + }); + case LAYER_TYPE.MVT_VECTOR: + return new MvtVectorLayer({ + layerDescriptor: layerDescriptor as VectorLayerDescriptor, + source: source as IVectorSource, + joins: createJoinInstances( + layerDescriptor as VectorLayerDescriptor, + source as IVectorSource + ), + customIcons, + }); + default: + throw new Error(`Unrecognized layerType ${layerDescriptor.type}`); + } + } catch (error) { + return new InvalidLayer(layerDescriptor, error); } } diff --git a/x-pack/plugins/ml/public/application/services/field_format_service.ts b/x-pack/plugins/ml/public/application/services/field_format_service.ts index a43e134f84cfb..8519a13e7d7bc 100644 --- a/x-pack/plugins/ml/public/application/services/field_format_service.ts +++ b/x-pack/plugins/ml/public/application/services/field_format_service.ts @@ -8,16 +8,20 @@ import { mlFunctionToESAggregation } from '../../../common/util/job_utils'; import { getDataViewById, getDataViewIdFromName } from '../util/index_utils'; import { mlJobService } from './job_service'; +import type { MlIndexUtils } from '../util/index_service'; +import type { MlApiServices } from './ml_api_service'; type FormatsByJobId = Record; type IndexPatternIdsByJob = Record; // Service for accessing FieldFormat objects configured for a Kibana data view // for use in formatting the actual and typical values from anomalies. -class FieldFormatService { +export class FieldFormatService { indexPatternIdsByJob: IndexPatternIdsByJob = {}; formatsByJob: FormatsByJobId = {}; + constructor(private mlApiServices?: MlApiServices, private mlIndexUtils?: MlIndexUtils) {} + // Populate the service with the FieldFormats for the list of jobs with the // specified IDs. List of Kibana data views is passed, with a title // attribute set in each pattern which will be compared to the indices @@ -32,10 +36,17 @@ class FieldFormatService { ( await Promise.all( jobIds.map(async (jobId) => { - const jobObj = mlJobService.getJob(jobId); + const getDataViewId = this.mlIndexUtils?.getDataViewIdFromName ?? getDataViewIdFromName; + let jobObj; + if (this.mlApiServices) { + const { jobs } = await this.mlApiServices.getJobs({ jobId }); + jobObj = jobs[0]; + } else { + jobObj = mlJobService.getJob(jobId); + } return { jobId, - dataViewId: await getDataViewIdFromName(jobObj.datafeed_config.indices.join(',')), + dataViewId: await getDataViewId(jobObj.datafeed_config!.indices.join(',')), }; }) ) @@ -68,41 +79,40 @@ class FieldFormatService { } } - getFormatsForJob(jobId: string): Promise { - return new Promise((resolve, reject) => { - const jobObj = mlJobService.getJob(jobId); - const detectors = jobObj.analysis_config.detectors || []; - const formatsByDetector: any[] = []; + async getFormatsForJob(jobId: string): Promise { + let jobObj; + const getDataView = this.mlIndexUtils?.getDataViewById ?? getDataViewById; + if (this.mlApiServices) { + const { jobs } = await this.mlApiServices.getJobs({ jobId }); + jobObj = jobs[0]; + } else { + jobObj = mlJobService.getJob(jobId); + } + const detectors = jobObj.analysis_config.detectors || []; + const formatsByDetector: any[] = []; - const dataViewId = this.indexPatternIdsByJob[jobId]; - if (dataViewId !== undefined) { - // Load the full data view configuration to obtain the formats of each field. - getDataViewById(dataViewId) - .then((dataView) => { - // Store the FieldFormat for each job by detector_index. - const fieldList = dataView.fields; - detectors.forEach((dtr) => { - const esAgg = mlFunctionToESAggregation(dtr.function); - // distinct_count detectors should fall back to the default - // formatter as the values are just counts. - if (dtr.field_name !== undefined && esAgg !== 'cardinality') { - const field = fieldList.getByName(dtr.field_name); - if (field !== undefined) { - formatsByDetector[dtr.detector_index!] = dataView.getFormatterForField(field); - } - } - }); + const dataViewId = this.indexPatternIdsByJob[jobId]; + if (dataViewId !== undefined) { + // Load the full data view configuration to obtain the formats of each field. + const dataView = await getDataView(dataViewId); + // Store the FieldFormat for each job by detector_index. + const fieldList = dataView.fields; + detectors.forEach((dtr) => { + const esAgg = mlFunctionToESAggregation(dtr.function); + // distinct_count detectors should fall back to the default + // formatter as the values are just counts. + if (dtr.field_name !== undefined && esAgg !== 'cardinality') { + const field = fieldList.getByName(dtr.field_name); + if (field !== undefined) { + formatsByDetector[dtr.detector_index!] = dataView.getFormatterForField(field); + } + } + }); + } - resolve(formatsByDetector); - }) - .catch((err) => { - reject(err); - }); - } else { - resolve(formatsByDetector); - } - }); + return formatsByDetector; } } export const mlFieldFormatService = new FieldFormatService(); +export type MlFieldFormatService = typeof mlFieldFormatService; diff --git a/x-pack/plugins/ml/public/application/services/field_format_service_factory.ts b/x-pack/plugins/ml/public/application/services/field_format_service_factory.ts new file mode 100644 index 0000000000000..daefab69154c5 --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/field_format_service_factory.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type MlFieldFormatService, FieldFormatService } from './field_format_service'; +import type { MlIndexUtils } from '../util/index_service'; +import type { MlApiServices } from './ml_api_service'; + +export function fieldFormatServiceFactory( + mlApiServices: MlApiServices, + mlIndexUtils: MlIndexUtils +): MlFieldFormatService { + return new FieldFormatService(mlApiServices, mlIndexUtils); +} diff --git a/x-pack/plugins/ml/public/application/services/forecast_service.d.ts b/x-pack/plugins/ml/public/application/services/forecast_service.d.ts index 0bfd8f56385d6..55df37b2307da 100644 --- a/x-pack/plugins/ml/public/application/services/forecast_service.d.ts +++ b/x-pack/plugins/ml/public/application/services/forecast_service.d.ts @@ -32,3 +32,5 @@ export const mlForecastService: { getForecastDateRange: (job: Job, forecastId: string) => Promise; }; + +export type MlForecastService = typeof mlForecastService; diff --git a/x-pack/plugins/ml/public/application/services/forecast_service_provider.ts b/x-pack/plugins/ml/public/application/services/forecast_service_provider.ts new file mode 100644 index 0000000000000..c776a79a6f475 --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/forecast_service_provider.ts @@ -0,0 +1,395 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Service for carrying out requests to run ML forecasts and to obtain +// data on forecasts that have been performed. +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { get, find, each } from 'lodash'; +import { map } from 'rxjs/operators'; +import type { MlApiServices } from './ml_api_service'; +import type { Job } from '../../../common/types/anomaly_detection_jobs'; + +export interface AggType { + avg: string; + max: string; + min: string; +} + +// TODO Consolidate with legacy code in +// `x-pack/plugins/ml/public/application/services/forecast_service.js` and +// `x-pack/plugins/ml/public/application/services/forecast_service.d.ts`. +export function forecastServiceProvider(mlApiServices: MlApiServices) { + return { + // Gets a basic summary of the most recently run forecasts for the specified + // job, with results at or later than the supplied timestamp. + // Extra query object can be supplied, or pass null if no additional query. + // Returned response contains a forecasts property, which is an array of objects + // containing id, earliest and latest keys. + getForecastsSummary(job: Job, query: any, earliestMs: number, maxResults: any) { + return new Promise((resolve, reject) => { + const obj: { success: boolean; forecasts: Record } = { + success: true, + forecasts: [], + }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the job ID, result type and earliest time, plus + // the additional query if supplied. + const filterCriteria = [ + { + term: { result_type: 'model_forecast_request_stats' }, + }, + { + term: { job_id: job.job_id }, + }, + { + range: { + timestamp: { + gte: earliestMs, + format: 'epoch_millis', + }, + }, + }, + ]; + + if (query) { + filterCriteria.push(query); + } + + mlApiServices.results + .anomalySearch( + { + // @ts-expect-error SearchRequest type has not been updated to include size + size: maxResults, + body: { + query: { + bool: { + filter: filterCriteria, + }, + }, + sort: [{ forecast_create_timestamp: { order: 'desc' } }], + }, + }, + [job.job_id] + ) + .then((resp) => { + if (resp.hits.total.value > 0) { + obj.forecasts = resp.hits.hits.map((hit) => hit._source); + } + + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); + }); + }, + // Obtains the earliest and latest timestamps for the forecast data from + // the forecast with the specified ID. + // Returned response contains earliest and latest properties which are the + // timestamps of the first and last model_forecast results. + getForecastDateRange(job: Job, forecastId: string) { + return new Promise((resolve, reject) => { + const obj = { + success: true, + earliest: null, + latest: null, + }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the job ID, forecast ID, result type and time range. + const filterCriteria = [ + { + query_string: { + query: 'result_type:model_forecast', + analyze_wildcard: true, + }, + }, + { + term: { job_id: job.job_id }, + }, + { + term: { forecast_id: forecastId }, + }, + ]; + + // TODO - add in criteria for detector index and entity fields (by, over, partition) + // once forecasting with these parameters is supported. + + mlApiServices.results + .anomalySearch( + { + // @ts-expect-error SearchRequest type has not been updated to include size + size: 0, + body: { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: { + earliest: { + min: { + field: 'timestamp', + }, + }, + latest: { + max: { + field: 'timestamp', + }, + }, + }, + }, + }, + [job.job_id] + ) + .then((resp) => { + obj.earliest = get(resp, 'aggregations.earliest.value', null); + obj.latest = get(resp, 'aggregations.latest.value', null); + if (obj.earliest === null || obj.latest === null) { + reject(resp); + } else { + resolve(obj); + } + }) + .catch((resp) => { + reject(resp); + }); + }); + }, + // Obtains the requested forecast model data for the forecast with the specified ID. + getForecastData( + job: Job, + detectorIndex: number, + forecastId: string, + entityFields: any, + earliestMs: number, + latestMs: number, + intervalMs: number, + aggType?: AggType + ) { + // Extract the partition, by, over fields on which to filter. + const criteriaFields = []; + const detector = job.analysis_config.detectors[detectorIndex]; + if (detector.partition_field_name !== undefined) { + const partitionEntity = find(entityFields, { fieldName: detector.partition_field_name }); + if (partitionEntity !== undefined) { + criteriaFields.push( + { fieldName: 'partition_field_name', fieldValue: partitionEntity.fieldName }, + { fieldName: 'partition_field_value', fieldValue: partitionEntity.fieldValue } + ); + } + } + + if (detector.over_field_name !== undefined) { + const overEntity = find(entityFields, { fieldName: detector.over_field_name }); + if (overEntity !== undefined) { + criteriaFields.push( + { fieldName: 'over_field_name', fieldValue: overEntity.fieldName }, + { fieldName: 'over_field_value', fieldValue: overEntity.fieldValue } + ); + } + } + + if (detector.by_field_name !== undefined) { + const byEntity = find(entityFields, { fieldName: detector.by_field_name }); + if (byEntity !== undefined) { + criteriaFields.push( + { fieldName: 'by_field_name', fieldValue: byEntity.fieldName }, + { fieldName: 'by_field_value', fieldValue: byEntity.fieldValue } + ); + } + } + + const obj: { success: boolean; results: Record } = { + success: true, + results: {}, + }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the job ID, forecast ID, detector index, result type and time range. + const filterCriteria: estypes.QueryDslQueryContainer[] = [ + { + query_string: { + query: 'result_type:model_forecast', + analyze_wildcard: true, + }, + }, + { + term: { job_id: job.job_id }, + }, + { + term: { forecast_id: forecastId }, + }, + { + term: { detector_index: detectorIndex }, + }, + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, + }, + ]; + + // Add in term queries for each of the specified criteria. + each(criteriaFields, (criteria) => { + filterCriteria.push({ + term: { + [criteria.fieldName]: criteria.fieldValue, + }, + }); + }); + + // If an aggType object has been passed in, use it. + // Otherwise default to avg, min and max aggs for the + // forecast prediction, upper and lower + const forecastAggs = + aggType === undefined + ? { avg: 'avg', max: 'max', min: 'min' } + : { + avg: aggType.avg, + max: aggType.max, + min: aggType.min, + }; + + return mlApiServices.results + .anomalySearch$( + { + // @ts-expect-error SearchRequest type has not been updated to include size + size: 0, + body: { + query: { + bool: { + filter: filterCriteria, + }, + }, + aggs: { + times: { + date_histogram: { + field: 'timestamp', + fixed_interval: `${intervalMs}ms`, + min_doc_count: 1, + }, + aggs: { + prediction: { + [forecastAggs.avg]: { + field: 'forecast_prediction', + }, + }, + forecastUpper: { + [forecastAggs.max]: { + field: 'forecast_upper', + }, + }, + forecastLower: { + [forecastAggs.min]: { + field: 'forecast_lower', + }, + }, + }, + }, + }, + }, + }, + [job.job_id] + ) + .pipe( + map((resp) => { + const aggregationsByTime = get(resp, ['aggregations', 'times', 'buckets'], []); + each(aggregationsByTime, (dataForTime) => { + const time = dataForTime.key; + obj.results[time] = { + prediction: get(dataForTime, ['prediction', 'value']), + forecastUpper: get(dataForTime, ['forecastUpper', 'value']), + forecastLower: get(dataForTime, ['forecastLower', 'value']), + }; + }); + + return obj; + }) + ); + }, + // Runs a forecast + runForecast(jobId: string, duration?: string) { + // eslint-disable-next-line no-console + console.log('ML forecast service run forecast with duration:', duration); + return new Promise((resolve, reject) => { + mlApiServices + .forecast({ + jobId, + duration, + }) + .then((resp) => { + resolve(resp); + }) + .catch((err) => { + reject(err); + }); + }); + }, + // Gets stats for a forecast that has been run on the specified job. + // Returned response contains a stats property, including + // forecast_progress (a value from 0 to 1), + // and forecast_status ('finished' when complete) properties. + getForecastRequestStats(job: Job, forecastId: string) { + return new Promise((resolve, reject) => { + const obj = { + success: true, + stats: {}, + }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the job ID, result type and earliest time. + const filterCriteria = [ + { + query_string: { + query: 'result_type:model_forecast_request_stats', + analyze_wildcard: true, + }, + }, + { + term: { job_id: job.job_id }, + }, + { + term: { forecast_id: forecastId }, + }, + ]; + + mlApiServices.results + .anomalySearch( + { + // @ts-expect-error SearchRequest type has not been updated to include size + size: 1, + body: { + query: { + bool: { + filter: filterCriteria, + }, + }, + }, + }, + [job.job_id] + ) + .then((resp) => { + if (resp.hits.total.value > 0) { + obj.stats = resp.hits.hits[0]._source; + } + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); + }); + }, + }; +} + +export type MlForecastService = ReturnType; diff --git a/x-pack/plugins/ml/public/application/services/results_service/index.ts b/x-pack/plugins/ml/public/application/services/results_service/index.ts index 4fe6b7add2a6b..883b54dd73e72 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/index.ts @@ -5,9 +5,11 @@ * 2.0. */ +import { useMemo } from 'react'; import { resultsServiceRxProvider } from './result_service_rx'; import { resultsServiceProvider } from './results_service'; import { ml, MlApiServices } from '../ml_api_service'; +import { useMlKibana } from '../../contexts/kibana'; export type MlResultsService = typeof mlResultsService; @@ -29,3 +31,14 @@ export function mlResultsServiceProvider(mlApiServices: MlApiServices) { ...resultsServiceRxProvider(mlApiServices), }; } + +export function useMlResultsService(): MlResultsService { + const { + services: { + mlServices: { mlApiServices }, + }, + } = useMlKibana(); + + const resultsService = useMemo(() => mlResultsServiceProvider(mlApiServices), [mlApiServices]); + return resultsService; +} diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/plot_function_controls/plot_function_controls.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/plot_function_controls/plot_function_controls.tsx index c707bbee2c5b9..493e74755588b 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/plot_function_controls/plot_function_controls.tsx +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/plot_function_controls/plot_function_controls.tsx @@ -9,9 +9,11 @@ import React, { useCallback, useEffect } from 'react'; import { EuiFlexItem, EuiFormRow, EuiSelect } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { ML_JOB_AGGREGATION } from '@kbn/ml-anomaly-utils'; +import { MlJob } from '@elastic/elasticsearch/lib/api/types'; import { mlJobService } from '../../../services/job_service'; import { getFunctionDescription, isMetricDetector } from '../../get_function_description'; import { useToastNotificationService } from '../../../services/toast_notification_service'; +import { useMlResultsService } from '../../../services/results_service'; import type { CombinedJob } from '../../../../../common/types/anomaly_detection_jobs'; const plotByFunctionOptions = [ @@ -36,6 +38,7 @@ const plotByFunctionOptions = [ ]; export const PlotByFunctionControls = ({ functionDescription, + job, setFunctionDescription, selectedDetectorIndex, selectedJobId, @@ -43,6 +46,7 @@ export const PlotByFunctionControls = ({ entityControlsCount, }: { functionDescription: undefined | string; + job?: CombinedJob | MlJob; setFunctionDescription: (func: string) => void; selectedDetectorIndex: number; selectedJobId: string; @@ -50,6 +54,7 @@ export const PlotByFunctionControls = ({ entityControlsCount: number; }) => { const toastNotificationService = useToastNotificationService(); + const mlResultsService = useMlResultsService(); const getFunctionDescriptionToPlot = useCallback( async ( @@ -65,18 +70,19 @@ export const PlotByFunctionControls = ({ selectedJobId: _selectedJobId, selectedJob: _selectedJob, }, - toastNotificationService + toastNotificationService, + mlResultsService ); setFunctionDescription(functionToPlot); }, - [setFunctionDescription, toastNotificationService] + [setFunctionDescription, toastNotificationService, mlResultsService] ); useEffect(() => { if (functionDescription !== undefined) { return; } - const selectedJob = mlJobService.getJob(selectedJobId); + const selectedJob = (job ?? mlJobService.getJob(selectedJobId)) as CombinedJob; // if no controls, it's okay to fetch // if there are series controls, only fetch if user has selected something const validEntities = diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/series_controls/series_controls.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/series_controls/series_controls.tsx index 23bc2f80eb1a8..666d56f15fbc8 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/series_controls/series_controls.tsx +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/series_controls/series_controls.tsx @@ -12,9 +12,10 @@ import { debounce } from 'lodash'; import { lastValueFrom } from 'rxjs'; import { useStorage } from '@kbn/ml-local-storage'; import type { MlEntityFieldType } from '@kbn/ml-anomaly-utils'; +import { MlJob } from '@elastic/elasticsearch/lib/api/types'; import { EntityControl } from '../entity_control'; import { mlJobService } from '../../../services/job_service'; -import { Detector, JobId } from '../../../../../common/types/anomaly_detection_jobs'; +import { CombinedJob, Detector, JobId } from '../../../../../common/types/anomaly_detection_jobs'; import { useMlKibana } from '../../../contexts/kibana'; import { APP_STATE_ACTION } from '../../timeseriesexplorer_constants'; import { @@ -67,12 +68,13 @@ const getDefaultFieldConfig = ( }; interface SeriesControlsProps { - selectedDetectorIndex: number; - selectedJobId: JobId; - bounds: any; appStateHandler: Function; + bounds: any; + functionDescription?: string; + job?: CombinedJob | MlJob; + selectedDetectorIndex: number; selectedEntities: Record; - functionDescription: string; + selectedJobId: JobId; setFunctionDescription: (func: string) => void; } @@ -80,13 +82,14 @@ interface SeriesControlsProps { * Component for handling the detector and entities controls. */ export const SeriesControls: FC = ({ - bounds, - selectedDetectorIndex, - selectedJobId, appStateHandler, + bounds, children, - selectedEntities, functionDescription, + job, + selectedDetectorIndex, + selectedEntities, + selectedJobId, setFunctionDescription, }) => { const { @@ -97,7 +100,11 @@ export const SeriesControls: FC = ({ }, } = useMlKibana(); - const selectedJob = useMemo(() => mlJobService.getJob(selectedJobId), [selectedJobId]); + const selectedJob: CombinedJob | MlJob = useMemo( + () => job ?? mlJobService.getJob(selectedJobId), + // eslint-disable-next-line react-hooks/exhaustive-deps + [selectedJobId] + ); const isModelPlotEnabled = !!selectedJob.model_plot_config?.enabled; @@ -108,11 +115,17 @@ export const SeriesControls: FC = ({ index: number; detector_description: Detector['detector_description']; }> = useMemo(() => { - return getViewableDetectors(selectedJob); + return getViewableDetectors(selectedJob as CombinedJob); }, [selectedJob]); const entityControls = useMemo(() => { - return getControlsForDetector(selectedDetectorIndex, selectedEntities, selectedJobId); + return getControlsForDetector( + selectedDetectorIndex, + selectedEntities, + selectedJobId, + selectedJob as CombinedJob + ); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [selectedDetectorIndex, selectedEntities, selectedJobId]); const [storageFieldsConfig, setStorageFieldsConfig] = useStorage< @@ -318,6 +331,7 @@ export const SeriesControls: FC = ({ ); })} `anomaly-marker multi-bucket ${getSeverityWithLow(d.anomalyScore).id}`); // Add rectangular markers for any scheduled events. - const scheduledEventMarkers = d3 + const scheduledEventMarkers = chartElement .select('.focus-chart-markers') .selectAll('.scheduled-event-marker') .data(data.filter((d) => d.scheduledEvents !== undefined)); @@ -898,7 +915,7 @@ class TimeseriesChartIntl extends Component { .attr('d', this.focusValuesLine(focusForecastData)) .classed('hidden', !showForecast); - const forecastDots = d3 + const forecastDots = chartElement .select('.focus-chart-markers.forecast') .selectAll('.metric-value') .data(focusForecastData); @@ -1007,7 +1024,7 @@ class TimeseriesChartIntl extends Component { const chartElement = d3.select(this.rootNode); chartElement.selectAll('.focus-zoom a').on('click', function () { d3.event.preventDefault(); - setZoomInterval(d3.select(this).attr('data-ms')); + setZoomInterval(this.getAttribute('data-ms')); }); } @@ -1129,7 +1146,7 @@ class TimeseriesChartIntl extends Component { .attr('y2', brushChartHeight); // Add x axis. - const timeBuckets = getTimeBucketsFromCache(); + const timeBuckets = this.getTimeBuckets(); timeBuckets.setInterval('auto'); timeBuckets.setBounds(bounds); const xAxisTickFormat = timeBuckets.getScaledDateFormat(); @@ -1328,6 +1345,7 @@ class TimeseriesChartIntl extends Component {
`); + const that = this; function brushing() { const brushExtent = brush.extent(); mask.reveal(brushExtent); @@ -1345,11 +1363,11 @@ class TimeseriesChartIntl extends Component { topBorder.attr('width', topBorderWidth); const isEmpty = brush.empty(); - d3.selectAll('.brush-handle').style('visibility', isEmpty ? 'hidden' : 'visible'); + const chartElement = d3.select(that.rootNode); + chartElement.selectAll('.brush-handle').style('visibility', isEmpty ? 'hidden' : 'visible'); } brushing(); - const that = this; function brushed() { const isEmpty = brush.empty(); const selectedBounds = isEmpty ? contextXScale.domain() : brush.extent(); @@ -1478,18 +1496,19 @@ class TimeseriesChartIntl extends Component { // Sets the extent of the brush on the context chart to the // supplied from and to Date objects. setContextBrushExtent = (from, to) => { + const chartElement = d3.select(this.rootNode); const brush = this.brush; const brushExtent = brush.extent(); const newExtent = [from, to]; brush.extent(newExtent); - brush(d3.select('.brush')); + brush(chartElement.select('.brush')); if ( newExtent[0].getTime() !== brushExtent[0].getTime() || newExtent[1].getTime() !== brushExtent[1].getTime() ) { - brush.event(d3.select('.brush')); + brush.event(chartElement.select('.brush')); } }; @@ -1867,12 +1886,13 @@ class TimeseriesChartIntl extends Component { anomalyTime, focusAggregationInterval ); + const chartElement = d3.select(this.rootNode); // Render an additional highlighted anomaly marker on the focus chart. // TODO - plot anomaly markers for cases where there is an anomaly due // to the absence of data and model plot is enabled. if (markerToSelect !== undefined) { - const selectedMarker = d3 + const selectedMarker = chartElement .select('.focus-chart-markers') .selectAll('.focus-chart-highlighted-marker') .data([markerToSelect]); @@ -1905,7 +1925,6 @@ class TimeseriesChartIntl extends Component { // Display the chart tooltip for this marker. // Note the values of the record and marker may differ depending on the levels of aggregation. - const chartElement = d3.select(this.rootNode); const anomalyMarker = chartElement.selectAll( '.focus-chart-markers .anomaly-marker.highlighted' ); @@ -1916,7 +1935,8 @@ class TimeseriesChartIntl extends Component { } unhighlightFocusChartAnomaly() { - d3.select('.focus-chart-markers').selectAll('.anomaly-marker.highlighted').remove(); + const chartElement = d3.select(this.rootNode); + chartElement.select('.focus-chart-markers').selectAll('.anomaly-marker.highlighted').remove(); this.props.tooltipService.hide(); } diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/timeseries_chart/timeseries_chart_with_tooltip.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/timeseries_chart/timeseries_chart_with_tooltip.tsx index 66da1e4222887..b9e09158bf280 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/timeseries_chart/timeseries_chart_with_tooltip.tsx +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/timeseries_chart/timeseries_chart_with_tooltip.tsx @@ -15,7 +15,7 @@ import { CombinedJob } from '../../../../../common/types/anomaly_detection_jobs' import { ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE } from '../../../../../common/constants/search'; import { Annotation } from '../../../../../common/types/annotations'; import { useMlKibana, useNotifications } from '../../../contexts/kibana'; -import { getBoundsRoundedToInterval } from '../../../util/time_buckets'; +import { useTimeBucketsService } from '../../../util/time_buckets_service'; import { getControlsForDetector } from '../../get_controls_for_detector'; import { MlAnnotationUpdatesContext } from '../../../contexts/ml/ml_annotation_updates_context'; import { SourceIndicesWithGeoFields } from '../../../explorer/explorer_utils'; @@ -23,6 +23,7 @@ import { SourceIndicesWithGeoFields } from '../../../explorer/explorer_utils'; interface TimeSeriesChartWithTooltipsProps { bounds: any; detectorIndex: number; + embeddableMode?: boolean; renderFocusChartOnly: boolean; selectedJob: CombinedJob; selectedEntities: Record; @@ -41,6 +42,7 @@ interface TimeSeriesChartWithTooltipsProps { export const TimeSeriesChartWithTooltips: FC = ({ bounds, detectorIndex, + embeddableMode, renderFocusChartOnly, selectedJob, selectedEntities, @@ -80,13 +82,19 @@ export const TimeSeriesChartWithTooltips: FC = // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + const mlTimeBucketsService = useTimeBucketsService(); + useEffect(() => { let unmounted = false; const entities = getControlsForDetector(detectorIndex, selectedEntities, selectedJob.job_id); const nonBlankEntities = Array.isArray(entities) ? entities.filter((entity) => entity.fieldValue !== null) : undefined; - const searchBounds = getBoundsRoundedToInterval(bounds, contextAggregationInterval, false); + const searchBounds = mlTimeBucketsService.getBoundsRoundedToInterval( + bounds, + contextAggregationInterval, + false + ); /** * Loads the full list of annotations for job without any aggs or time boundaries @@ -138,6 +146,7 @@ export const TimeSeriesChartWithTooltips: FC = annotationData={annotationData} bounds={bounds} detectorIndex={detectorIndex} + embeddableMode={embeddableMode} renderFocusChartOnly={renderFocusChartOnly} selectedJob={selectedJob} showAnnotations={showAnnotations} diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/get_controls_for_detector.ts b/x-pack/plugins/ml/public/application/timeseriesexplorer/get_controls_for_detector.ts index cf8e1f0aa989c..30f097dabb8ab 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/get_controls_for_detector.ts +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/get_controls_for_detector.ts @@ -7,7 +7,7 @@ import { mlJobService } from '../services/job_service'; import { Entity } from './components/entity_control/entity_control'; -import { JobId } from '../../../common/types/anomaly_detection_jobs'; +import type { JobId, CombinedJob } from '../../../common/types/anomaly_detection_jobs'; /** * Extracts entities from the detector configuration @@ -15,9 +15,10 @@ import { JobId } from '../../../common/types/anomaly_detection_jobs'; export function getControlsForDetector( selectedDetectorIndex: number, selectedEntities: Record, - selectedJobId: JobId + selectedJobId: JobId, + job?: CombinedJob ): Entity[] { - const selectedJob = mlJobService.getJob(selectedJobId); + const selectedJob = job ?? mlJobService.getJob(selectedJobId); const entities: Entity[] = []; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/get_function_description.ts b/x-pack/plugins/ml/public/application/timeseriesexplorer/get_function_description.ts index d0dfdc9ed372b..e6f1a2ec65afd 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/get_function_description.ts +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/get_function_description.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; import { lastValueFrom } from 'rxjs'; import { ES_AGGREGATION, ML_JOB_AGGREGATION } from '@kbn/ml-anomaly-utils'; -import { mlResultsService } from '../services/results_service'; +import { type MlResultsService } from '../services/results_service'; import { ToastNotificationService } from '../services/toast_notification_service'; import { getControlsForDetector } from './get_controls_for_detector'; import { getCriteriaFields } from './get_criteria_fields'; @@ -41,7 +41,8 @@ export const getFunctionDescription = async ( selectedJobId: string; selectedJob: CombinedJob; }, - toastNotificationService: ToastNotificationService + toastNotificationService: ToastNotificationService, + mlResultsService: MlResultsService ) => { // if the detector's function is metric, fetch the highest scoring anomaly record // and set to plot the function_description (avg/min/max) of that record by default diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js index 757f4cb06543e..ad3f71e5df22d 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js @@ -77,6 +77,7 @@ import { processMetricPlotResults, processRecordScoreResults, getFocusData, + getTimeseriesexplorerDefaultState, } from './timeseriesexplorer_utils'; import { ANOMALY_DETECTION_DEFAULT_TIME_RANGE } from '../../../common/constants/settings'; import { getControlsForDetector } from './get_controls_for_detector'; @@ -96,46 +97,6 @@ const allValuesLabel = i18n.translate('xpack.ml.timeSeriesExplorer.allPartitionV defaultMessage: 'all', }); -function getTimeseriesexplorerDefaultState() { - return { - chartDetails: undefined, - contextAggregationInterval: undefined, - contextChartData: undefined, - contextForecastData: undefined, - // Not chartable if e.g. model plot with terms for a varp detector - dataNotChartable: false, - entitiesLoading: false, - entityValues: {}, - focusAnnotationData: [], - focusAggregationInterval: {}, - focusChartData: undefined, - focusForecastData: undefined, - fullRefresh: true, - hasResults: false, - // Counter to keep track of what data sets have been loaded. - loadCounter: 0, - loading: false, - modelPlotEnabled: false, - // Toggles display of annotations in the focus chart - showAnnotations: true, - showAnnotationsCheckbox: true, - // Toggles display of forecast data in the focus chart - showForecast: true, - showForecastCheckbox: false, - // Toggles display of model bounds in the focus chart - showModelBounds: true, - showModelBoundsCheckbox: false, - svgWidth: 0, - tableData: undefined, - zoomFrom: undefined, - zoomTo: undefined, - zoomFromFocusLoaded: undefined, - zoomToFocusLoaded: undefined, - chartDataError: undefined, - sourceIndicesWithGeoFields: {}, - }; -} - const containerPadding = 34; export class TimeSeriesExplorer extends React.Component { @@ -265,7 +226,7 @@ export class TimeSeriesExplorer extends React.Component { } /** - * Gets focus data for the current component state/ + * Gets focus data for the current component state */ getFocusData(selection) { const { selectedJobId, selectedForecastId, selectedDetectorIndex, functionDescription } = @@ -745,7 +706,6 @@ export class TimeSeriesExplorer extends React.Component { ); } } - // Required to redraw the time series chart when the container is resized. this.resizeChecker = new ResizeChecker(this.resizeRef.current); this.resizeChecker.on('resize', () => { @@ -1091,7 +1051,6 @@ export class TimeSeriesExplorer extends React.Component { entities={entityControls} /> )} - {arePartitioningFieldsProvided && jobs.length > 0 && (fullRefresh === false || loading === false) && diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_constants.ts b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_constants.ts index d66dca5f565d7..5d13c73f8401f 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_constants.ts +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_constants.ts @@ -17,7 +17,9 @@ export const APP_STATE_ACTION = { SET_ZOOM: 'SET_ZOOM', UNSET_ZOOM: 'UNSET_ZOOM', SET_FUNCTION_DESCRIPTION: 'SET_FUNCTION_DESCRIPTION', -}; +} as const; + +export type TimeseriesexplorerActionType = typeof APP_STATE_ACTION[keyof typeof APP_STATE_ACTION]; export const CHARTS_POINT_TARGET = 500; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/index.ts b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/index.ts new file mode 100644 index 0000000000000..b81b4bc96a434 --- /dev/null +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { TimeSeriesExplorerEmbeddableChart } from './timeseriesexplorer_embeddable_chart'; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/timeseriesexplorer_checkbox.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/timeseriesexplorer_checkbox.tsx new file mode 100644 index 0000000000000..e1136e54180ac --- /dev/null +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/timeseriesexplorer_checkbox.tsx @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useMemo } from 'react'; +import { EuiCheckbox, EuiFlexItem, htmlIdGenerator } from '@elastic/eui'; + +interface Props { + id: string; + label: string; + checked: boolean; + onChange: (e: React.ChangeEvent) => void; +} + +export const TimeseriesExplorerCheckbox: FC = ({ id, label, checked, onChange }) => { + const checkboxId = useMemo(() => `id-${htmlIdGenerator()()}`, []); + return ( + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/timeseriesexplorer_embeddable_chart.js b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/timeseriesexplorer_embeddable_chart.js new file mode 100644 index 0000000000000..fd6b0239199bc --- /dev/null +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_embeddable_chart/timeseriesexplorer_embeddable_chart.js @@ -0,0 +1,897 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * React component for rendering Single Metric Viewer. + */ + +import { isEqual } from 'lodash'; +import moment from 'moment-timezone'; +import { Subject, Subscription, forkJoin } from 'rxjs'; +import { debounceTime, switchMap, tap, withLatestFrom } from 'rxjs/operators'; + +import PropTypes from 'prop-types'; +import React, { Fragment } from 'react'; + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { context } from '@kbn/kibana-react-plugin/public'; + +import { + EuiCallOut, + EuiCheckbox, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiTitle, + EuiTextColor, +} from '@elastic/eui'; +import { TimeSeriesExplorerHelpPopover } from '../timeseriesexplorer_help_popover'; + +import { + isModelPlotEnabled, + isModelPlotChartableForDetector, + isSourceDataChartableForDetector, +} from '../../../../common/util/job_utils'; + +import { LoadingIndicator } from '../../components/loading_indicator/loading_indicator'; +import { TimeseriesexplorerNoChartData } from '../components/timeseriesexplorer_no_chart_data'; + +import { + APP_STATE_ACTION, + CHARTS_POINT_TARGET, + TIME_FIELD_NAME, +} from '../timeseriesexplorer_constants'; +import { getControlsForDetector } from '../get_controls_for_detector'; +import { TimeSeriesChartWithTooltips } from '../components/timeseries_chart/timeseries_chart_with_tooltip'; +import { aggregationTypeTransform } from '@kbn/ml-anomaly-utils'; +import { isMetricDetector } from '../get_function_description'; +import { TimeseriesexplorerChartDataError } from '../components/timeseriesexplorer_chart_data_error'; +import { TimeseriesExplorerCheckbox } from './timeseriesexplorer_checkbox'; +import { timeBucketsServiceFactory } from '../../util/time_buckets_service'; +import { timeSeriesExplorerServiceFactory } from '../../util/time_series_explorer_service'; +import { getTimeseriesexplorerDefaultState } from '../timeseriesexplorer_utils'; + +// Used to indicate the chart is being plotted across +// all partition field values, where the cardinality of the field cannot be +// obtained as it is not aggregatable e.g. 'all distinct kpi_indicator values' +const allValuesLabel = i18n.translate('xpack.ml.timeSeriesExplorer.allPartitionValuesLabel', { + defaultMessage: 'all', +}); + +export class TimeSeriesExplorerEmbeddableChart extends React.Component { + static propTypes = { + appStateHandler: PropTypes.func.isRequired, + autoZoomDuration: PropTypes.number.isRequired, + bounds: PropTypes.object.isRequired, + chartWidth: PropTypes.number.isRequired, + lastRefresh: PropTypes.number.isRequired, + previousRefresh: PropTypes.number.isRequired, + selectedJobId: PropTypes.string.isRequired, + selectedDetectorIndex: PropTypes.number, + selectedEntities: PropTypes.object, + selectedForecastId: PropTypes.string, + zoom: PropTypes.object, + toastNotificationService: PropTypes.object, + dataViewsService: PropTypes.object, + }; + + state = getTimeseriesexplorerDefaultState(); + + subscriptions = new Subscription(); + + unmounted = false; + + /** + * Subject for listening brush time range selection. + */ + contextChart$ = new Subject(); + + /** + * Access ML services in react context. + */ + static contextType = context; + + getBoundsRoundedToInterval; + mlTimeSeriesExplorer; + + /** + * Returns field names that don't have a selection yet. + */ + getFieldNamesWithEmptyValues = () => { + const latestEntityControls = this.getControlsForDetector(); + return latestEntityControls + .filter(({ fieldValue }) => fieldValue === null) + .map(({ fieldName }) => fieldName); + }; + + /** + * Checks if all entity control dropdowns have a selection. + */ + arePartitioningFieldsProvided = () => { + const fieldNamesWithEmptyValues = this.getFieldNamesWithEmptyValues(); + return fieldNamesWithEmptyValues.length === 0; + }; + + toggleShowAnnotationsHandler = () => { + this.setState((prevState) => ({ + showAnnotations: !prevState.showAnnotations, + })); + }; + + toggleShowForecastHandler = () => { + this.setState((prevState) => ({ + showForecast: !prevState.showForecast, + })); + }; + + toggleShowModelBoundsHandler = () => { + this.setState({ + showModelBounds: !this.state.showModelBounds, + }); + }; + + setFunctionDescription = (selectedFuction) => { + this.props.appStateHandler(APP_STATE_ACTION.SET_FUNCTION_DESCRIPTION, selectedFuction); + }; + + previousChartProps = {}; + previousShowAnnotations = undefined; + previousShowForecast = undefined; + previousShowModelBounds = undefined; + + tableFilter = (field, value, operator) => { + const entities = this.getControlsForDetector(); + const entity = entities.find(({ fieldName }) => fieldName === field); + + if (entity === undefined) { + return; + } + + const { appStateHandler } = this.props; + + let resultValue = ''; + if (operator === '+' && entity.fieldValue !== value) { + resultValue = value; + } else if (operator === '-' && entity.fieldValue === value) { + resultValue = null; + } else { + return; + } + + const resultEntities = { + ...entities.reduce((appStateEntities, appStateEntity) => { + appStateEntities[appStateEntity.fieldName] = appStateEntity.fieldValue; + return appStateEntities; + }, {}), + [entity.fieldName]: resultValue, + }; + + appStateHandler(APP_STATE_ACTION.SET_ENTITIES, resultEntities); + }; + + contextChartSelectedInitCallDone = false; + + getFocusAggregationInterval(selection) { + const { selectedJob } = this.props; + + // Calculate the aggregation interval for the focus chart. + const bounds = { min: moment(selection.from), max: moment(selection.to) }; + + return this.mlTimeSeriesExplorer.calculateAggregationInterval( + bounds, + CHARTS_POINT_TARGET, + selectedJob + ); + } + + /** + * Gets focus data for the current component state + */ + getFocusData(selection) { + const { selectedForecastId, selectedDetectorIndex, functionDescription, selectedJob } = + this.props; + const { modelPlotEnabled } = this.state; + if (isMetricDetector(selectedJob, selectedDetectorIndex) && functionDescription === undefined) { + return; + } + const entityControls = this.getControlsForDetector(); + + // Calculate the aggregation interval for the focus chart. + const bounds = { min: moment(selection.from), max: moment(selection.to) }; + const focusAggregationInterval = this.getFocusAggregationInterval(selection); + + // Ensure the search bounds align to the bucketing interval so that the first and last buckets are complete. + // For sum or count detectors, short buckets would hold smaller values, and model bounds would also be affected + // to some extent with all detector functions if not searching complete buckets. + const searchBounds = this.getBoundsRoundedToInterval(bounds, focusAggregationInterval, false); + + return this.mlTimeSeriesExplorer.getFocusData( + this.getCriteriaFields(selectedDetectorIndex, entityControls), + selectedDetectorIndex, + focusAggregationInterval, + selectedForecastId, + modelPlotEnabled, + entityControls.filter((entity) => entity.fieldValue !== null), + searchBounds, + selectedJob, + functionDescription, + TIME_FIELD_NAME + ); + } + + contextChartSelected = (selection) => { + const zoomState = { + from: selection.from.toISOString(), + to: selection.to.toISOString(), + }; + + if ( + isEqual(this.props.zoom, zoomState) && + this.state.focusChartData !== undefined && + this.props.previousRefresh === this.props.lastRefresh + ) { + return; + } + + this.contextChart$.next(selection); + this.props.appStateHandler(APP_STATE_ACTION.SET_ZOOM, zoomState); + }; + + setForecastId = (forecastId) => { + this.props.appStateHandler(APP_STATE_ACTION.SET_FORECAST_ID, forecastId); + }; + + displayErrorToastMessages = (error, errorMsg) => { + if (this.props.toastNotificationService) { + this.props.toastNotificationService.displayErrorToast(error, errorMsg, 2000); + } + this.setState({ loading: false, chartDataError: errorMsg }); + }; + + loadSingleMetricData = (fullRefresh = true) => { + const { + autoZoomDuration, + bounds, + selectedDetectorIndex, + zoom, + functionDescription, + selectedJob, + } = this.props; + + const { loadCounter: currentLoadCounter } = this.state; + if (selectedJob === undefined) { + return; + } + if (isMetricDetector(selectedJob, selectedDetectorIndex) && functionDescription === undefined) { + return; + } + + const functionToPlotByIfMetric = aggregationTypeTransform.toES(functionDescription); + + this.contextChartSelectedInitCallDone = false; + + // Only when `fullRefresh` is true we'll reset all data + // and show the loading spinner within the page. + const entityControls = this.getControlsForDetector(); + this.setState( + { + fullRefresh, + loadCounter: currentLoadCounter + 1, + loading: true, + chartDataError: undefined, + ...(fullRefresh + ? { + chartDetails: undefined, + contextChartData: undefined, + contextForecastData: undefined, + focusChartData: undefined, + focusForecastData: undefined, + modelPlotEnabled: + isModelPlotChartableForDetector(selectedJob, selectedDetectorIndex) && + isModelPlotEnabled(selectedJob, selectedDetectorIndex, entityControls), + hasResults: false, + dataNotChartable: false, + } + : {}), + }, + () => { + const { loadCounter, modelPlotEnabled } = this.state; + const { selectedJob } = this.props; + + const detectorIndex = selectedDetectorIndex; + + let awaitingCount = 3; + + const stateUpdate = {}; + + // finish() function, called after each data set has been loaded and processed. + // The last one to call it will trigger the page render. + const finish = (counterVar) => { + awaitingCount--; + if (awaitingCount === 0 && counterVar === loadCounter) { + stateUpdate.hasResults = + (Array.isArray(stateUpdate.contextChartData) && + stateUpdate.contextChartData.length > 0) || + (Array.isArray(stateUpdate.contextForecastData) && + stateUpdate.contextForecastData.length > 0); + stateUpdate.loading = false; + + // Set zoomFrom/zoomTo attributes in scope which will result in the metric chart automatically + // selecting the specified range in the context chart, and so loading that date range in the focus chart. + // Only touch the zoom range if data for the context chart has been loaded and all necessary + // partition fields have a selection. + if ( + stateUpdate.contextChartData.length && + this.arePartitioningFieldsProvided() === true + ) { + // Check for a zoom parameter in the appState (URL). + let focusRange = this.mlTimeSeriesExplorer.calculateInitialFocusRange( + zoom, + stateUpdate.contextAggregationInterval, + bounds + ); + if ( + focusRange === undefined || + this.previousSelectedForecastId !== this.props.selectedForecastId + ) { + focusRange = this.mlTimeSeriesExplorer.calculateDefaultFocusRange( + autoZoomDuration, + stateUpdate.contextAggregationInterval, + stateUpdate.contextChartData, + stateUpdate.contextForecastData + ); + this.previousSelectedForecastId = this.props.selectedForecastId; + } + + this.contextChartSelected({ + from: focusRange[0], + to: focusRange[1], + }); + } + + this.setState(stateUpdate); + } + }; + + const nonBlankEntities = entityControls.filter((entity) => { + return entity.fieldValue !== null; + }); + + if ( + modelPlotEnabled === false && + isSourceDataChartableForDetector(selectedJob, detectorIndex) === false && + nonBlankEntities.length > 0 + ) { + // For detectors where model plot has been enabled with a terms filter and the + // selected entity(s) are not in the terms list, indicate that data cannot be viewed. + stateUpdate.hasResults = false; + stateUpdate.loading = false; + stateUpdate.dataNotChartable = true; + this.setState(stateUpdate); + return; + } + + // Calculate the aggregation interval for the context chart. + // Context chart swimlane will display bucket anomaly score at the same interval. + stateUpdate.contextAggregationInterval = + this.mlTimeSeriesExplorer.calculateAggregationInterval( + bounds, + CHARTS_POINT_TARGET, + selectedJob + ); + + // Ensure the search bounds align to the bucketing interval so that the first and last buckets are complete. + // For sum or count detectors, short buckets would hold smaller values, and model bounds would also be affected + // to some extent with all detector functions if not searching complete buckets. + const searchBounds = this.getBoundsRoundedToInterval( + bounds, + stateUpdate.contextAggregationInterval, + false + ); + + // Query 1 - load metric data at low granularity across full time range. + // Pass a counter flag into the finish() function to make sure we only process the results + // for the most recent call to the load the data in cases where the job selection and time filter + // have been altered in quick succession (such as from the job picker with 'Apply time range'). + const counter = loadCounter; + this.context.services.mlServices.mlTimeSeriesSearchService + .getMetricData( + selectedJob, + detectorIndex, + nonBlankEntities, + searchBounds.min.valueOf(), + searchBounds.max.valueOf(), + stateUpdate.contextAggregationInterval.asMilliseconds(), + functionToPlotByIfMetric + ) + .toPromise() + .then((resp) => { + const fullRangeChartData = this.mlTimeSeriesExplorer.processMetricPlotResults( + resp.results, + modelPlotEnabled + ); + stateUpdate.contextChartData = fullRangeChartData; + finish(counter); + }) + .catch((err) => { + const errorMsg = i18n.translate('xpack.ml.timeSeriesExplorer.metricDataErrorMessage', { + defaultMessage: 'Error getting metric data', + }); + this.displayErrorToastMessages(err, errorMsg); + }); + + // Query 2 - load max record score at same granularity as context chart + // across full time range for use in the swimlane. + this.context.services.mlServices.mlResultsService + .getRecordMaxScoreByTime( + selectedJob.job_id, + this.getCriteriaFields(detectorIndex, entityControls), + searchBounds.min.valueOf(), + searchBounds.max.valueOf(), + stateUpdate.contextAggregationInterval.asMilliseconds(), + functionToPlotByIfMetric + ) + .then((resp) => { + const fullRangeRecordScoreData = this.mlTimeSeriesExplorer.processRecordScoreResults( + resp.results + ); + stateUpdate.swimlaneData = fullRangeRecordScoreData; + finish(counter); + }) + .catch((err) => { + const errorMsg = i18n.translate( + 'xpack.ml.timeSeriesExplorer.bucketAnomalyScoresErrorMessage', + { + defaultMessage: 'Error getting bucket anomaly scores', + } + ); + + this.displayErrorToastMessages(err, errorMsg); + }); + + // Query 3 - load details on the chart used in the chart title (charting function and entity(s)). + this.context.services.mlServices.mlTimeSeriesSearchService + .getChartDetails( + selectedJob, + detectorIndex, + entityControls, + searchBounds.min.valueOf(), + searchBounds.max.valueOf() + ) + .then((resp) => { + stateUpdate.chartDetails = resp.results; + finish(counter); + }) + .catch((err) => { + this.displayErrorToastMessages( + err, + i18n.translate('xpack.ml.timeSeriesExplorer.entityCountsErrorMessage', { + defaultMessage: 'Error getting entity counts', + }) + ); + }); + } + ); + }; + + /** + * Updates local state of detector related controls from the global state. + * @param callback to invoke after a state update. + */ + getControlsForDetector = () => { + const { selectedDetectorIndex, selectedEntities, selectedJobId, selectedJob } = this.props; + return getControlsForDetector( + selectedDetectorIndex, + selectedEntities, + selectedJobId, + selectedJob + ); + }; + + /** + * Updates criteria fields for API calls, e.g. getAnomaliesTableData + * @param detectorIndex + * @param entities + */ + getCriteriaFields(detectorIndex, entities) { + // Only filter on the entity if the field has a value. + const nonBlankEntities = entities.filter((entity) => entity.fieldValue !== null); + return [ + { + fieldName: 'detector_index', + fieldValue: detectorIndex, + }, + ...nonBlankEntities, + ]; + } + + async componentDidMount() { + this.getBoundsRoundedToInterval = timeBucketsServiceFactory( + this.context.services.uiSettings + ).getBoundsRoundedToInterval; + + this.mlTimeSeriesExplorer = timeSeriesExplorerServiceFactory( + this.context.services.uiSettings, + this.context.services.mlServices.mlApiServices, + this.context.services.mlServices.mlResultsService + ); + + // Listen for context chart updates. + this.subscriptions.add( + this.contextChart$ + .pipe( + tap((selection) => { + this.setState({ + zoomFrom: selection.from, + zoomTo: selection.to, + }); + }), + debounceTime(500), + tap((selection) => { + const { + contextChartData, + contextForecastData, + focusChartData, + zoomFromFocusLoaded, + zoomToFocusLoaded, + } = this.state; + + if ( + (contextChartData === undefined || contextChartData.length === 0) && + (contextForecastData === undefined || contextForecastData.length === 0) + ) { + return; + } + + if ( + (this.contextChartSelectedInitCallDone === false && focusChartData === undefined) || + zoomFromFocusLoaded.getTime() !== selection.from.getTime() || + zoomToFocusLoaded.getTime() !== selection.to.getTime() + ) { + this.contextChartSelectedInitCallDone = true; + + this.setState({ + loading: true, + fullRefresh: false, + }); + } + }), + switchMap((selection) => { + return forkJoin([this.getFocusData(selection)]); + }), + withLatestFrom(this.contextChart$) + ) + .subscribe(([[refreshFocusData, tableData], selection]) => { + const { modelPlotEnabled } = this.state; + + // All the data is ready now for a state update. + this.setState({ + focusAggregationInterval: this.getFocusAggregationInterval({ + from: selection.from, + to: selection.to, + }), + loading: false, + showModelBoundsCheckbox: modelPlotEnabled && refreshFocusData.focusChartData.length > 0, + zoomFromFocusLoaded: selection.from, + zoomToFocusLoaded: selection.to, + ...refreshFocusData, + ...tableData, + }); + }) + ); + + if (this.context && this.props.selectedJob !== undefined) { + // Populate the map of jobs / detectors / field formatters for the selected IDs and refresh. + this.context.services.mlServices.mlFieldFormatService.populateFormats([ + this.props.selectedJob.job_id, + ]); + } + + this.componentDidUpdate(); + } + + componentDidUpdate(previousProps) { + if ( + previousProps === undefined || + previousProps.selectedForecastId !== this.props.selectedForecastId + ) { + if (this.props.selectedForecastId !== undefined) { + // Ensure the forecast data will be shown if hidden previously. + this.setState({ showForecast: true }); + // Not best practice but we need the previous value for another comparison + // once all the data was loaded. + if (previousProps !== undefined) { + this.previousSelectedForecastId = previousProps.selectedForecastId; + } + } + } + + if ( + previousProps === undefined || + !isEqual(previousProps.bounds, this.props.bounds) || + (!isEqual(previousProps.lastRefresh, this.props.lastRefresh) && + previousProps.lastRefresh !== 0) || + !isEqual(previousProps.selectedDetectorIndex, this.props.selectedDetectorIndex) || + !isEqual(previousProps.selectedEntities, this.props.selectedEntities) || + previousProps.selectedForecastId !== this.props.selectedForecastId || + previousProps.selectedJobId !== this.props.selectedJobId || + previousProps.functionDescription !== this.props.functionDescription + ) { + const fullRefresh = + previousProps === undefined || + !isEqual(previousProps.bounds, this.props.bounds) || + !isEqual(previousProps.selectedDetectorIndex, this.props.selectedDetectorIndex) || + !isEqual(previousProps.selectedEntities, this.props.selectedEntities) || + previousProps.selectedForecastId !== this.props.selectedForecastId || + previousProps.selectedJobId !== this.props.selectedJobId || + previousProps.functionDescription !== this.props.functionDescription; + this.loadSingleMetricData(fullRefresh); + } + + if (previousProps === undefined) { + return; + } + } + + componentWillUnmount() { + this.subscriptions.unsubscribe(); + this.unmounted = true; + } + + render() { + const { + autoZoomDuration, + bounds, + chartWidth, + lastRefresh, + selectedDetectorIndex, + selectedJob, + } = this.props; + + const { + chartDetails, + contextAggregationInterval, + contextChartData, + contextForecastData, + dataNotChartable, + focusAggregationInterval, + focusAnnotationData, + focusChartData, + focusForecastData, + fullRefresh, + hasResults, + loading, + modelPlotEnabled, + showAnnotations, + showAnnotationsCheckbox, + showForecast, + showForecastCheckbox, + showModelBounds, + showModelBoundsCheckbox, + swimlaneData, + zoomFrom, + zoomTo, + zoomFromFocusLoaded, + zoomToFocusLoaded, + chartDataError, + } = this.state; + const chartProps = { + modelPlotEnabled, + contextChartData, + contextChartSelected: this.contextChartSelected, + contextForecastData, + contextAggregationInterval, + swimlaneData, + focusAnnotationData, + focusChartData, + focusForecastData, + focusAggregationInterval, + svgWidth: chartWidth, + zoomFrom, + zoomTo, + zoomFromFocusLoaded, + zoomToFocusLoaded, + autoZoomDuration, + }; + + const entityControls = this.getControlsForDetector(); + const fieldNamesWithEmptyValues = this.getFieldNamesWithEmptyValues(); + const arePartitioningFieldsProvided = this.arePartitioningFieldsProvided(); + + let renderFocusChartOnly = true; + + if ( + isEqual(this.previousChartProps.focusForecastData, chartProps.focusForecastData) && + isEqual(this.previousChartProps.focusChartData, chartProps.focusChartData) && + isEqual(this.previousChartProps.focusAnnotationData, chartProps.focusAnnotationData) && + this.previousShowForecast === showForecast && + this.previousShowModelBounds === showModelBounds && + this.props.previousRefresh === lastRefresh + ) { + renderFocusChartOnly = false; + } + + this.previousChartProps = chartProps; + this.previousShowForecast = showForecast; + this.previousShowModelBounds = showModelBounds; + + return ( + <> + {fieldNamesWithEmptyValues.length > 0 && ( + <> + + } + iconType="help" + size="s" + /> + + + )} + + {fullRefresh && loading === true && ( + + )} + + {loading === false && chartDataError !== undefined && ( + + )} + + {arePartitioningFieldsProvided && + selectedJob && + (fullRefresh === false || loading === false) && + hasResults === false && + chartDataError === undefined && ( + + )} + {arePartitioningFieldsProvided && + selectedJob && + (fullRefresh === false || loading === false) && + hasResults === true && ( +
+ + + +

+ + {i18n.translate( + 'xpack.ml.timeSeriesExplorer.singleTimeSeriesAnalysisTitle', + { + defaultMessage: 'Single time series analysis of {functionLabel}', + values: { functionLabel: chartDetails.functionLabel }, + } + )} + +   + {chartDetails.entityData.count === 1 && ( + + {chartDetails.entityData.entities.length > 0 && '('} + {chartDetails.entityData.entities + .map((entity) => { + return `${entity.fieldName}: ${entity.fieldValue}`; + }) + .join(', ')} + {chartDetails.entityData.entities.length > 0 && ')'} + + )} + {chartDetails.entityData.count !== 1 && ( + + {chartDetails.entityData.entities.map((countData, i) => { + return ( + + {i18n.translate( + 'xpack.ml.timeSeriesExplorer.countDataInChartDetailsDescription', + { + defaultMessage: + '{openBrace}{cardinalityValue} distinct {fieldName} {cardinality, plural, one {} other { values}}{closeBrace}', + values: { + openBrace: i === 0 ? '(' : '', + closeBrace: + i === chartDetails.entityData.entities.length - 1 + ? ')' + : '', + cardinalityValue: + countData.cardinality === 0 + ? allValuesLabel + : countData.cardinality, + cardinality: countData.cardinality, + fieldName: countData.fieldName, + }, + } + )} + {i !== chartDetails.entityData.entities.length - 1 ? ', ' : ''} + + ); + })} + + )} +

+ + + + + + + + + {showModelBoundsCheckbox && ( + + )} + + {showAnnotationsCheckbox && ( + + )} + + {showForecastCheckbox && ( + + + {i18n.translate('xpack.ml.timeSeriesExplorer.showForecastLabel', { + defaultMessage: 'show forecast', + })} + + } + checked={showForecast} + onChange={this.toggleShowForecastHandler} + /> + + )} + + + +
+ )} + + ); + } +} diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_help_popover.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_help_popover.tsx index afd93fd5acee1..3557523c113fc 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_help_popover.tsx +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_help_popover.tsx @@ -5,12 +5,14 @@ * 2.0. */ -import React from 'react'; +import React, { FC } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { HelpPopover } from '../components/help_popover/help_popover'; -export const TimeSeriesExplorerHelpPopover = () => { +export const TimeSeriesExplorerHelpPopover: FC<{ embeddableMode: boolean }> = ({ + embeddableMode, +}) => { return ( { defaultMessage="If you create a forecast, predicted data values are added to the chart. A shaded area around these values represents the confidence level; as you forecast further into the future, the confidence level generally decreases." />

-

- -

+ {!embeddableMode && ( +

+ +

+ )}

{ + if ( + isModelPlotChartableForDetector(job, detectorIndex) && + isModelPlotEnabled(job, detectorIndex, entityFields) + ) { + // Extract the partition, by, over fields on which to filter. + const criteriaFields = []; + const detector = job.analysis_config.detectors[detectorIndex]; + if (detector.partition_field_name !== undefined) { + const partitionEntity: any = find(entityFields, { + fieldName: detector.partition_field_name, + }); + if (partitionEntity !== undefined) { + criteriaFields.push( + { fieldName: 'partition_field_name', fieldValue: partitionEntity.fieldName }, + { fieldName: 'partition_field_value', fieldValue: partitionEntity.fieldValue } + ); + } + } + + if (detector.over_field_name !== undefined) { + const overEntity: any = find(entityFields, { fieldName: detector.over_field_name }); + if (overEntity !== undefined) { + criteriaFields.push( + { fieldName: 'over_field_name', fieldValue: overEntity.fieldName }, + { fieldName: 'over_field_value', fieldValue: overEntity.fieldValue } + ); + } + } + + if (detector.by_field_name !== undefined) { + const byEntity: any = find(entityFields, { fieldName: detector.by_field_name }); + if (byEntity !== undefined) { + criteriaFields.push( + { fieldName: 'by_field_name', fieldValue: byEntity.fieldName }, + { fieldName: 'by_field_value', fieldValue: byEntity.fieldValue } + ); + } + } + + return mlResultsService.getModelPlotOutput( + job.job_id, + detectorIndex, + criteriaFields, + earliestMs, + latestMs, + intervalMs + ); + } else { + const obj: ModelPlotOutput = { + success: true, + results: {}, + }; + + const chartConfig = buildConfigFromDetector(job, detectorIndex); + + return mlResultsService + .getMetricData( + chartConfig.datafeedConfig.indices.join(','), + entityFields, + chartConfig.datafeedConfig.query, + esMetricFunction ?? chartConfig.metricFunction, + chartConfig.metricFieldName, + chartConfig.summaryCountFieldName, + chartConfig.timeField, + earliestMs, + latestMs, + intervalMs, + chartConfig?.datafeedConfig + ) + .pipe( + map((resp) => { + each(resp.results, (value, time) => { + // @ts-ignore + obj.results[time] = { + actual: value, + }; + }); + return obj; + }) + ); + } + }, + // Builds chart detail information (charting function description and entity counts) used + // in the title area of the time series chart. + // Queries Elasticsearch if necessary to obtain the distinct count of entities + // for which data is being plotted. + getChartDetails( + job: Job, + detectorIndex: number, + entityFields: any[], + earliestMs: number, + latestMs: number + ) { + return new Promise((resolve, reject) => { + const obj: any = { + success: true, + results: { functionLabel: '', entityData: { entities: [] } }, + }; + + const chartConfig = buildConfigFromDetector(job, detectorIndex); + let functionLabel: string | null = chartConfig.metricFunction; + if (chartConfig.metricFieldName !== undefined) { + functionLabel += ' '; + functionLabel += chartConfig.metricFieldName; + } + obj.results.functionLabel = functionLabel; + + const blankEntityFields = filter(entityFields, (entity) => { + return entity.fieldValue === null; + }); + + // Look to see if any of the entity fields have defined values + // (i.e. blank input), and if so obtain the cardinality. + if (blankEntityFields.length === 0) { + obj.results.entityData.count = 1; + obj.results.entityData.entities = entityFields; + resolve(obj); + } else { + const entityFieldNames: string[] = blankEntityFields.map((f) => f.fieldName); + mlApiServices + .getCardinalityOfFields({ + index: chartConfig.datafeedConfig.indices.join(','), + fieldNames: entityFieldNames, + query: chartConfig.datafeedConfig.query, + timeFieldName: chartConfig.timeField, + earliestMs, + latestMs, + }) + .then((results: any) => { + each(blankEntityFields, (field) => { + // results will not contain keys for non-aggregatable fields, + // so store as 0 to indicate over all field values. + obj.results.entityData.entities.push({ + fieldName: field.fieldName, + cardinality: get(results, field.fieldName, 0), + }); + }); + + resolve(obj); + }) + .catch((resp: any) => { + reject(resp); + }); + } + }); + }, + }; +} + +export type MlTimeSeriesSeachService = ReturnType; diff --git a/x-pack/plugins/ml/public/application/util/index_service.ts b/x-pack/plugins/ml/public/application/util/index_service.ts new file mode 100644 index 0000000000000..f4b6a1fc13d77 --- /dev/null +++ b/x-pack/plugins/ml/public/application/util/index_service.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/public'; +import type { Job } from '../../../common/types/anomaly_detection_jobs'; + +// TODO Consolidate with legacy code in `ml/public/application/util/index_utils.ts`. +export function indexServiceFactory(dataViewsService: DataViewsContract) { + return { + /** + * Retrieves the data view ID from the given name. + * If a job is passed in, a temporary data view will be created if the requested data view doesn't exist. + * @param name - The name or index pattern of the data view. + * @param job - Optional job object. + * @returns The data view ID or null if it doesn't exist. + */ + async getDataViewIdFromName(name: string, job?: Job): Promise { + if (dataViewsService === null) { + throw new Error('Data views are not initialized!'); + } + const dataViews = await dataViewsService.find(name); + const dataView = dataViews.find((dv) => dv.getIndexPattern() === name); + if (!dataView) { + if (job !== undefined) { + const tempDataView = await dataViewsService.create({ + id: undefined, + name, + title: name, + timeFieldName: job.data_description.time_field!, + }); + return tempDataView.id ?? null; + } + return null; + } + return dataView.id ?? dataView.getIndexPattern(); + }, + getDataViewById(id: string): Promise { + if (dataViewsService === null) { + throw new Error('Data views are not initialized!'); + } + + if (id) { + return dataViewsService.get(id); + } else { + return dataViewsService.create({}); + } + }, + }; +} + +export type MlIndexUtils = ReturnType; diff --git a/x-pack/plugins/ml/public/application/util/time_buckets.d.ts b/x-pack/plugins/ml/public/application/util/time_buckets.d.ts index 9a5410918a099..0f413ed9c2c71 100644 --- a/x-pack/plugins/ml/public/application/util/time_buckets.d.ts +++ b/x-pack/plugins/ml/public/application/util/time_buckets.d.ts @@ -33,6 +33,7 @@ export declare class TimeBuckets { public setBounds(bounds: TimeRangeBounds): void; public getBounds(): { min: any; max: any }; public getInterval(): TimeBucketsInterval; + public getIntervalToNearestMultiple(divisorSecs: any): TimeBucketsInterval; public getScaledDateFormat(): string; } diff --git a/x-pack/plugins/ml/public/application/util/time_buckets_service.ts b/x-pack/plugins/ml/public/application/util/time_buckets_service.ts new file mode 100644 index 0000000000000..480f279a603b1 --- /dev/null +++ b/x-pack/plugins/ml/public/application/util/time_buckets_service.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import type { IUiSettingsClient } from '@kbn/core/public'; +import { UI_SETTINGS } from '@kbn/data-plugin/public'; +import moment from 'moment'; +import { type TimeRangeBounds, type TimeBucketsInterval, TimeBuckets } from './time_buckets'; +import { useMlKibana } from '../contexts/kibana'; + +// TODO Consolidate with legacy code in `ml/public/application/util/time_buckets.js`. +export function timeBucketsServiceFactory(uiSettings: IUiSettingsClient) { + function getTimeBuckets(): InstanceType { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + } + function getBoundsRoundedToInterval( + bounds: TimeRangeBounds, + interval: TimeBucketsInterval, + inclusiveEnd: boolean = false + ): Required { + // Returns new bounds, created by flooring the min of the provided bounds to the start of + // the specified interval (a moment duration), and rounded upwards (Math.ceil) to 1ms before + // the start of the next interval (Kibana dashboards search >= bounds min, and <= bounds max, + // so we subtract 1ms off the max to avoid querying start of the new Elasticsearch aggregation bucket). + const intervalMs = interval.asMilliseconds(); + const adjustedMinMs = Math.floor(bounds.min!.valueOf() / intervalMs) * intervalMs; + let adjustedMaxMs = Math.ceil(bounds.max!.valueOf() / intervalMs) * intervalMs; + + // Don't include the start ms of the next bucket unless specified.. + if (inclusiveEnd === false) { + adjustedMaxMs = adjustedMaxMs - 1; + } + return { min: moment(adjustedMinMs), max: moment(adjustedMaxMs) }; + } + + return { getTimeBuckets, getBoundsRoundedToInterval }; +} + +export type TimeBucketsService = ReturnType; + +export function useTimeBucketsService(): TimeBucketsService { + const { + services: { uiSettings }, + } = useMlKibana(); + + const mlTimeBucketsService = useMemo(() => timeBucketsServiceFactory(uiSettings), [uiSettings]); + return mlTimeBucketsService; +} diff --git a/x-pack/plugins/ml/public/application/util/time_series_explorer_service.ts b/x-pack/plugins/ml/public/application/util/time_series_explorer_service.ts new file mode 100644 index 0000000000000..4af8d98093cbd --- /dev/null +++ b/x-pack/plugins/ml/public/application/util/time_series_explorer_service.ts @@ -0,0 +1,648 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import type { IUiSettingsClient } from '@kbn/core/public'; +import { aggregationTypeTransform } from '@kbn/ml-anomaly-utils'; +import { isMultiBucketAnomaly, ML_JOB_AGGREGATION } from '@kbn/ml-anomaly-utils'; +import { extractErrorMessage } from '@kbn/ml-error-utils'; +import moment from 'moment'; +import { forkJoin, Observable, of } from 'rxjs'; +import { each, get } from 'lodash'; +import { catchError, map } from 'rxjs/operators'; +import { type MlAnomalyRecordDoc } from '@kbn/ml-anomaly-utils'; +import { parseInterval } from '../../../common/util/parse_interval'; +import type { GetAnnotationsResponse } from '../../../common/types/annotations'; +import { mlFunctionToESAggregation } from '../../../common/util/job_utils'; +import { ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE } from '../../../common/constants/search'; +import { CHARTS_POINT_TARGET } from '../timeseriesexplorer/timeseriesexplorer_constants'; +import { timeBucketsServiceFactory } from './time_buckets_service'; +import type { TimeRangeBounds } from './time_buckets'; +import type { Job } from '../../../common/types/anomaly_detection_jobs'; +import type { TimeBucketsInterval } from './time_buckets'; +import type { + ChartDataPoint, + FocusData, + Interval, +} from '../timeseriesexplorer/timeseriesexplorer_utils/get_focus_data'; +import type { CriteriaField } from '../services/results_service'; +import { + MAX_SCHEDULED_EVENTS, + TIME_FIELD_NAME, +} from '../timeseriesexplorer/timeseriesexplorer_constants'; +import type { MlApiServices } from '../services/ml_api_service'; +import { mlResultsServiceProvider, type MlResultsService } from '../services/results_service'; +import { forecastServiceProvider } from '../services/forecast_service_provider'; +import { timeSeriesSearchServiceFactory } from '../timeseriesexplorer/timeseriesexplorer_utils/time_series_search_service'; +import { useMlKibana } from '../contexts/kibana'; + +// TODO Consolidate with legacy code in +// `ml/public/application/timeseriesexplorer/timeseriesexplorer_utils/timeseriesexplorer_utils.js`. +export function timeSeriesExplorerServiceFactory( + uiSettings: IUiSettingsClient, + mlApiServices: MlApiServices, + mlResultsService: MlResultsService +) { + const timeBuckets = timeBucketsServiceFactory(uiSettings); + const mlForecastService = forecastServiceProvider(mlApiServices); + const mlTimeSeriesSearchService = timeSeriesSearchServiceFactory(mlResultsService, mlApiServices); + + function getAutoZoomDuration(selectedJob: Job) { + // Calculate the 'auto' zoom duration which shows data at bucket span granularity. + // Get the minimum bucket span of selected jobs. + let autoZoomDuration; + if (selectedJob.analysis_config.bucket_span) { + const bucketSpan = parseInterval(selectedJob.analysis_config.bucket_span); + const bucketSpanSeconds = bucketSpan!.asSeconds(); + + // In most cases the duration can be obtained by simply multiplying the points target + // Check that this duration returns the bucket span when run back through the + // TimeBucket interval calculation. + autoZoomDuration = bucketSpanSeconds * 1000 * (CHARTS_POINT_TARGET - 1); + + // Use a maxBars of 10% greater than the target. + const maxBars = Math.floor(1.1 * CHARTS_POINT_TARGET); + const buckets = timeBuckets.getTimeBuckets(); + buckets.setInterval('auto'); + buckets.setBarTarget(Math.floor(CHARTS_POINT_TARGET)); + buckets.setMaxBars(maxBars); + + // Set bounds from 'now' for testing the auto zoom duration. + const nowMs = new Date().getTime(); + const max = moment(nowMs); + const min = moment(nowMs - autoZoomDuration); + buckets.setBounds({ min, max }); + + const calculatedInterval = buckets.getIntervalToNearestMultiple(bucketSpanSeconds); + const calculatedIntervalSecs = calculatedInterval.asSeconds(); + if (calculatedIntervalSecs !== bucketSpanSeconds) { + // If we haven't got the span back, which may occur depending on the 'auto' ranges + // used in TimeBuckets and the bucket span of the job, then multiply by the ratio + // of the bucket span to the calculated interval. + autoZoomDuration = autoZoomDuration * (bucketSpanSeconds / calculatedIntervalSecs); + } + } + + return autoZoomDuration; + } + + function calculateAggregationInterval( + bounds: TimeRangeBounds, + bucketsTarget: number | undefined, + selectedJob: Job + ) { + // Aggregation interval used in queries should be a function of the time span of the chart + // and the bucket span of the selected job(s). + const barTarget = bucketsTarget !== undefined ? bucketsTarget : 100; + // Use a maxBars of 10% greater than the target. + const maxBars = Math.floor(1.1 * barTarget); + const buckets = timeBuckets.getTimeBuckets(); + buckets.setInterval('auto'); + buckets.setBounds(bounds); + buckets.setBarTarget(Math.floor(barTarget)); + buckets.setMaxBars(maxBars); + let aggInterval; + + if (selectedJob.analysis_config.bucket_span) { + // Ensure the aggregation interval is always a multiple of the bucket span to avoid strange + // behaviour such as adjacent chart buckets holding different numbers of job results. + const bucketSpan = parseInterval(selectedJob.analysis_config.bucket_span); + const bucketSpanSeconds = bucketSpan!.asSeconds(); + aggInterval = buckets.getIntervalToNearestMultiple(bucketSpanSeconds); + + // Set the interval back to the job bucket span if the auto interval is smaller. + const secs = aggInterval.asSeconds(); + if (secs < bucketSpanSeconds) { + buckets.setInterval(bucketSpanSeconds + 's'); + aggInterval = buckets.getInterval(); + } + } + + return aggInterval; + } + + function calculateInitialFocusRange( + zoomState: any, + contextAggregationInterval: any, + bounds: TimeRangeBounds + ) { + if (zoomState !== undefined) { + // Check that the zoom times are valid. + // zoomFrom must be at or after context chart search bounds earliest, + // zoomTo must be at or before context chart search bounds latest. + const zoomFrom = moment(zoomState.from, 'YYYY-MM-DDTHH:mm:ss.SSSZ', true); + const zoomTo = moment(zoomState.to, 'YYYY-MM-DDTHH:mm:ss.SSSZ', true); + const searchBounds = timeBuckets.getBoundsRoundedToInterval( + bounds, + contextAggregationInterval, + true + ); + const earliest = searchBounds.min; + const latest = searchBounds.max; + + if ( + zoomFrom.isValid() && + zoomTo.isValid() && + zoomTo.isAfter(zoomFrom) && + zoomFrom.isBetween(earliest, latest, null, '[]') && + zoomTo.isBetween(earliest, latest, null, '[]') + ) { + return [zoomFrom.toDate(), zoomTo.toDate()]; + } + } + + return undefined; + } + + function calculateDefaultFocusRange( + autoZoomDuration: any, + contextAggregationInterval: any, + contextChartData: any, + contextForecastData: any + ) { + const isForecastData = contextForecastData !== undefined && contextForecastData.length > 0; + + const combinedData = + isForecastData === false ? contextChartData : contextChartData.concat(contextForecastData); + const earliestDataDate = combinedData[0].date; + const latestDataDate = combinedData[combinedData.length - 1].date; + + let rangeEarliestMs; + let rangeLatestMs; + + if (isForecastData === true) { + // Return a range centred on the start of the forecast range, depending + // on the time range of the forecast and data. + const earliestForecastDataDate = contextForecastData[0].date; + const latestForecastDataDate = contextForecastData[contextForecastData.length - 1].date; + + rangeLatestMs = Math.min( + earliestForecastDataDate.getTime() + autoZoomDuration / 2, + latestForecastDataDate.getTime() + ); + rangeEarliestMs = Math.max(rangeLatestMs - autoZoomDuration, earliestDataDate.getTime()); + } else { + // Returns the range that shows the most recent data at bucket span granularity. + rangeLatestMs = latestDataDate.getTime() + contextAggregationInterval.asMilliseconds(); + rangeEarliestMs = Math.max(earliestDataDate.getTime(), rangeLatestMs - autoZoomDuration); + } + + return [new Date(rangeEarliestMs), new Date(rangeLatestMs)]; + } + + // Return dataset in format used by the swimlane. + // i.e. array of Objects with keys date (JavaScript date) and score. + function processRecordScoreResults(scoreData: any) { + const bucketScoreData: any = []; + each(scoreData, (dataForTime, time) => { + bucketScoreData.push({ + date: new Date(+time), + score: dataForTime.score, + }); + }); + + return bucketScoreData; + } + + // Return dataset in format used by the single metric chart. + // i.e. array of Objects with keys date (JavaScript date) and value, + // plus lower and upper keys if model plot is enabled for the series. + function processMetricPlotResults(metricPlotData: any, modelPlotEnabled: any) { + const metricPlotChartData: any = []; + if (modelPlotEnabled === true) { + each(metricPlotData, (dataForTime, time) => { + metricPlotChartData.push({ + date: new Date(+time), + lower: dataForTime.modelLower, + value: dataForTime.actual, + upper: dataForTime.modelUpper, + }); + }); + } else { + each(metricPlotData, (dataForTime, time) => { + metricPlotChartData.push({ + date: new Date(+time), + value: dataForTime.actual, + }); + }); + } + + return metricPlotChartData; + } + + // Returns forecast dataset in format used by the single metric chart. + // i.e. array of Objects with keys date (JavaScript date), isForecast, + // value, lower and upper keys. + function processForecastResults(forecastData: any) { + const forecastPlotChartData: any = []; + each(forecastData, (dataForTime, time) => { + forecastPlotChartData.push({ + date: new Date(+time), + isForecast: true, + lower: dataForTime.forecastLower, + value: dataForTime.prediction, + upper: dataForTime.forecastUpper, + }); + }); + + return forecastPlotChartData; + } + + // Finds the chart point which corresponds to an anomaly with the + // specified time. + function findChartPointForAnomalyTime( + chartData: any, + anomalyTime: any, + aggregationInterval: any + ) { + let chartPoint; + if (chartData === undefined) { + return chartPoint; + } + + for (let i = 0; i < chartData.length; i++) { + if (chartData[i].date.getTime() === anomalyTime) { + chartPoint = chartData[i]; + break; + } + } + + if (chartPoint === undefined) { + // Find the time of the point which falls immediately before the + // time of the anomaly. This is the start of the chart 'bucket' + // which contains the anomalous bucket. + let foundItem; + const intervalMs = aggregationInterval.asMilliseconds(); + for (let i = 0; i < chartData.length; i++) { + const itemTime = chartData[i].date.getTime(); + if (anomalyTime - itemTime < intervalMs) { + foundItem = chartData[i]; + break; + } + } + + chartPoint = foundItem; + } + + return chartPoint; + } + + // Uses data from the list of anomaly records to add anomalyScore, + // function, actual and typical properties, plus causes and multi-bucket + // info if applicable, to the chartData entries for anomalous buckets. + function processDataForFocusAnomalies( + chartData: ChartDataPoint[], + anomalyRecords: MlAnomalyRecordDoc[], + aggregationInterval: Interval, + modelPlotEnabled: boolean, + functionDescription?: string + ) { + const timesToAddPointsFor: number[] = []; + + // Iterate through the anomaly records making sure we have chart points for each anomaly. + const intervalMs = aggregationInterval.asMilliseconds(); + let lastChartDataPointTime: any; + if (chartData !== undefined && chartData.length > 0) { + lastChartDataPointTime = chartData[chartData.length - 1].date.getTime(); + } + anomalyRecords.forEach((record: MlAnomalyRecordDoc) => { + const recordTime = record[TIME_FIELD_NAME]; + const chartPoint = findChartPointForAnomalyTime(chartData, recordTime, aggregationInterval); + if (chartPoint === undefined) { + const timeToAdd = Math.floor(recordTime / intervalMs) * intervalMs; + if (timesToAddPointsFor.indexOf(timeToAdd) === -1 && timeToAdd !== lastChartDataPointTime) { + timesToAddPointsFor.push(timeToAdd); + } + } + }); + + timesToAddPointsFor.sort((a, b) => a - b); + + timesToAddPointsFor.forEach((time) => { + const pointToAdd: ChartDataPoint = { + date: new Date(time), + value: null, + }; + + if (modelPlotEnabled === true) { + pointToAdd.upper = null; + pointToAdd.lower = null; + } + chartData.push(pointToAdd); + }); + + // Iterate through the anomaly records adding the + // various properties required for display. + anomalyRecords.forEach((record) => { + // Look for a chart point with the same time as the record. + // If none found, find closest time in chartData set. + const recordTime = record[TIME_FIELD_NAME]; + if ( + record.function === ML_JOB_AGGREGATION.METRIC && + record.function_description !== functionDescription + ) + return; + + const chartPoint = findChartPointForAnomalyTime(chartData, recordTime, aggregationInterval); + if (chartPoint !== undefined) { + // If chart aggregation interval > bucket span, there may be more than + // one anomaly record in the interval, so use the properties from + // the record with the highest anomalyScore. + const recordScore = record.record_score; + const pointScore = chartPoint.anomalyScore; + if (pointScore === undefined || pointScore < recordScore) { + chartPoint.anomalyScore = recordScore; + chartPoint.function = record.function; + + if (record.actual !== undefined) { + // If cannot match chart point for anomaly time + // substitute the value with the record's actual so it won't plot as null/0 + if (chartPoint.value === null || record.function === ML_JOB_AGGREGATION.METRIC) { + chartPoint.value = Array.isArray(record.actual) ? record.actual[0] : record.actual; + } + + chartPoint.actual = record.actual; + chartPoint.typical = record.typical; + } else { + const causes = get(record, 'causes', []); + if (causes.length > 0) { + chartPoint.byFieldName = record.by_field_name; + chartPoint.numberOfCauses = causes.length; + if (causes.length === 1) { + // If only a single cause, copy actual and typical values to the top level. + const cause = record.causes![0]; + chartPoint.actual = cause.actual; + chartPoint.typical = cause.typical; + // substitute the value with the record's actual so it won't plot as null/0 + if (chartPoint.value === null) { + chartPoint.value = cause.actual; + } + } + } + } + + if ( + record.anomaly_score_explanation !== undefined && + record.anomaly_score_explanation.multi_bucket_impact !== undefined + ) { + chartPoint.multiBucketImpact = record.anomaly_score_explanation.multi_bucket_impact; + } + + chartPoint.isMultiBucketAnomaly = isMultiBucketAnomaly(record); + } + } + }); + + return chartData; + } + + function findChartPointForScheduledEvent(chartData: any, eventTime: any) { + let chartPoint; + if (chartData === undefined) { + return chartPoint; + } + + for (let i = 0; i < chartData.length; i++) { + if (chartData[i].date.getTime() === eventTime) { + chartPoint = chartData[i]; + break; + } + } + + return chartPoint; + } + // Adds a scheduledEvents property to any points in the chart data set + // which correspond to times of scheduled events for the job. + function processScheduledEventsForChart( + chartData: ChartDataPoint[], + scheduledEvents: Array<{ events: any; time: number }> | undefined, + aggregationInterval: TimeBucketsInterval + ) { + if (scheduledEvents !== undefined) { + const timesToAddPointsFor: number[] = []; + + // Iterate through the scheduled events making sure we have a chart point for each event. + const intervalMs = aggregationInterval.asMilliseconds(); + let lastChartDataPointTime: number | undefined; + if (chartData !== undefined && chartData.length > 0) { + lastChartDataPointTime = chartData[chartData.length - 1].date.getTime(); + } + + // In case there's no chart data/sparse data during these scheduled events + // ensure we add chart points at every aggregation interval for these scheduled events. + let sortRequired = false; + each(scheduledEvents, (events, time) => { + const exactChartPoint = findChartPointForScheduledEvent(chartData, +time); + + if (exactChartPoint !== undefined) { + exactChartPoint.scheduledEvents = events; + } else { + const timeToAdd: number = Math.floor(time / intervalMs) * intervalMs; + if ( + timesToAddPointsFor.indexOf(timeToAdd) === -1 && + timeToAdd !== lastChartDataPointTime + ) { + const pointToAdd = { + date: new Date(timeToAdd), + value: null, + scheduledEvents: events, + }; + + chartData.push(pointToAdd); + sortRequired = true; + } + } + }); + + // Sort chart data by time if extra points were added at the end of the array for scheduled events. + if (sortRequired) { + chartData.sort((a, b) => a.date.getTime() - b.date.getTime()); + } + } + + return chartData; + } + + function getFocusData( + criteriaFields: CriteriaField[], + detectorIndex: number, + focusAggregationInterval: TimeBucketsInterval, + forecastId: string, + modelPlotEnabled: boolean, + nonBlankEntities: any[], + searchBounds: any, + selectedJob: Job, + functionDescription?: string | undefined + ): Observable { + const esFunctionToPlotIfMetric = + functionDescription !== undefined + ? aggregationTypeTransform.toES(functionDescription) + : functionDescription; + + return forkJoin([ + // Query 1 - load metric data across selected time range. + mlTimeSeriesSearchService.getMetricData( + selectedJob, + detectorIndex, + nonBlankEntities, + searchBounds.min.valueOf(), + searchBounds.max.valueOf(), + focusAggregationInterval.asMilliseconds(), + esFunctionToPlotIfMetric + ), + // Query 2 - load all the records across selected time range for the chart anomaly markers. + mlApiServices.results.getAnomalyRecords$( + [selectedJob.job_id], + criteriaFields, + 0, + searchBounds.min.valueOf(), + searchBounds.max.valueOf(), + focusAggregationInterval.expression, + functionDescription + ), + // Query 3 - load any scheduled events for the selected job. + mlResultsService.getScheduledEventsByBucket( + [selectedJob.job_id], + searchBounds.min.valueOf(), + searchBounds.max.valueOf(), + focusAggregationInterval.asMilliseconds(), + 1, + MAX_SCHEDULED_EVENTS + ), + // Query 4 - load any annotations for the selected job. + mlApiServices.annotations + .getAnnotations$({ + jobIds: [selectedJob.job_id], + earliestMs: searchBounds.min.valueOf(), + latestMs: searchBounds.max.valueOf(), + maxAnnotations: ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE, + detectorIndex, + entities: nonBlankEntities, + }) + .pipe( + catchError((resp) => + of({ + annotations: {}, + totalCount: 0, + error: extractErrorMessage(resp), + success: false, + } as GetAnnotationsResponse) + ) + ), + // Plus query for forecast data if there is a forecastId stored in the appState. + forecastId !== undefined + ? (() => { + let aggType; + const detector = selectedJob.analysis_config.detectors[detectorIndex]; + const esAgg = mlFunctionToESAggregation(detector.function); + if (!modelPlotEnabled && (esAgg === 'sum' || esAgg === 'count')) { + aggType = { avg: 'sum', max: 'sum', min: 'sum' }; + } + return mlForecastService.getForecastData( + selectedJob, + detectorIndex, + forecastId, + nonBlankEntities, + searchBounds.min.valueOf(), + searchBounds.max.valueOf(), + focusAggregationInterval.asMilliseconds(), + aggType + ); + })() + : of(null), + ]).pipe( + map( + ([metricData, recordsForCriteria, scheduledEventsByBucket, annotations, forecastData]) => { + // Sort in descending time order before storing in scope. + const anomalyRecords = recordsForCriteria?.records + .sort((a, b) => a[TIME_FIELD_NAME] - b[TIME_FIELD_NAME]) + .reverse(); + + const scheduledEvents = scheduledEventsByBucket?.events[selectedJob.job_id]; + + let focusChartData = processMetricPlotResults(metricData.results, modelPlotEnabled); + // Tell the results container directives to render the focus chart. + focusChartData = processDataForFocusAnomalies( + focusChartData, + anomalyRecords, + focusAggregationInterval, + modelPlotEnabled, + functionDescription + ); + focusChartData = processScheduledEventsForChart( + focusChartData, + scheduledEvents, + focusAggregationInterval + ); + + const refreshFocusData: FocusData = { + scheduledEvents, + anomalyRecords, + focusChartData, + }; + + if (annotations) { + if (annotations.error !== undefined) { + refreshFocusData.focusAnnotationError = annotations.error; + refreshFocusData.focusAnnotationData = []; + } else { + refreshFocusData.focusAnnotationData = ( + annotations.annotations[selectedJob.job_id] ?? [] + ) + .sort((a, b) => { + return a.timestamp - b.timestamp; + }) + .map((d, i: number) => { + d.key = (i + 1).toString(); + return d; + }); + } + } + + if (forecastData) { + refreshFocusData.focusForecastData = processForecastResults(forecastData.results); + refreshFocusData.showForecastCheckbox = refreshFocusData.focusForecastData.length > 0; + } + return refreshFocusData; + } + ) + ); + } + + return { + getAutoZoomDuration, + calculateAggregationInterval, + calculateInitialFocusRange, + calculateDefaultFocusRange, + processRecordScoreResults, + processMetricPlotResults, + processForecastResults, + findChartPointForAnomalyTime, + processDataForFocusAnomalies, + findChartPointForScheduledEvent, + processScheduledEventsForChart, + getFocusData, + }; +} + +export function useTimeSeriesExplorerService(): TimeSeriesExplorerService { + const { + services: { + uiSettings, + mlServices: { mlApiServices }, + }, + } = useMlKibana(); + const mlResultsService = mlResultsServiceProvider(mlApiServices); + + const mlTimeSeriesExplorer = useMemo( + () => timeSeriesExplorerServiceFactory(uiSettings, mlApiServices, mlResultsService), + [uiSettings, mlApiServices, mlResultsService] + ); + return mlTimeSeriesExplorer; +} + +export type TimeSeriesExplorerService = ReturnType; diff --git a/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx b/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx index 00c4a02d4e929..182d070266c9a 100644 --- a/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx +++ b/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx @@ -26,7 +26,8 @@ import { JobSelectorFlyout } from './components/job_selector_flyout'; */ export async function resolveJobSelection( coreStart: CoreStart, - selectedJobIds?: JobId[] + selectedJobIds?: JobId[], + singleSelection: boolean = false ): Promise<{ jobIds: string[]; groups: Array<{ groupId: string; jobIds: string[] }> }> { const { http, @@ -74,7 +75,7 @@ export async function resolveJobSelection( selectedIds={selectedJobIds} withTimeRangeSelector={false} dateFormatTz={dateFormatTz} - singleSelection={false} + singleSelection={singleSelection} timeseriesOnly={true} onFlyoutClose={onFlyoutClose} onSelectionConfirmed={onSelectionConfirmed} diff --git a/x-pack/plugins/ml/public/embeddables/constants.ts b/x-pack/plugins/ml/public/embeddables/constants.ts index cfe50f25cd889..1001cd89c7498 100644 --- a/x-pack/plugins/ml/public/embeddables/constants.ts +++ b/x-pack/plugins/ml/public/embeddables/constants.ts @@ -7,6 +7,7 @@ export const ANOMALY_SWIMLANE_EMBEDDABLE_TYPE = 'ml_anomaly_swimlane' as const; export const ANOMALY_EXPLORER_CHARTS_EMBEDDABLE_TYPE = 'ml_anomaly_charts' as const; +export const ANOMALY_SINGLE_METRIC_VIEWER_EMBEDDABLE_TYPE = 'ml_single_metric_viewer' as const; export type AnomalySwimLaneEmbeddableType = typeof ANOMALY_SWIMLANE_EMBEDDABLE_TYPE; export type AnomalyExplorerChartsEmbeddableType = typeof ANOMALY_EXPLORER_CHARTS_EMBEDDABLE_TYPE; diff --git a/x-pack/plugins/ml/public/embeddables/index.ts b/x-pack/plugins/ml/public/embeddables/index.ts index 0a505fe04ea85..9f0d2d75b1162 100644 --- a/x-pack/plugins/ml/public/embeddables/index.ts +++ b/x-pack/plugins/ml/public/embeddables/index.ts @@ -9,6 +9,7 @@ import type { EmbeddableSetup } from '@kbn/embeddable-plugin/public'; import { AnomalySwimlaneEmbeddableFactory } from './anomaly_swimlane'; import type { MlCoreSetup } from '../plugin'; import { AnomalyChartsEmbeddableFactory } from './anomaly_charts'; +import { SingleMetricViewerEmbeddableFactory } from './single_metric_viewer'; export * from './constants'; export * from './types'; @@ -25,6 +26,8 @@ export function registerEmbeddables(embeddable: EmbeddableSetup, core: MlCoreSet ); const anomalyChartsFactory = new AnomalyChartsEmbeddableFactory(core.getStartServices); - embeddable.registerEmbeddableFactory(anomalyChartsFactory.type, anomalyChartsFactory); + + const singleMetricViewerFactory = new SingleMetricViewerEmbeddableFactory(core.getStartServices); + embeddable.registerEmbeddableFactory(singleMetricViewerFactory.type, singleMetricViewerFactory); } diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/_index.scss b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/_index.scss new file mode 100644 index 0000000000000..b6f91cc749dcc --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/_index.scss @@ -0,0 +1,6 @@ +// ML has it's own variables for coloring +@import '../../application/variables'; + +// Protect the rest of Kibana from ML generic namespacing +@import '../../application/timeseriesexplorer/timeseriesexplorer'; +@import '../../application/timeseriesexplorer/timeseriesexplorer_annotations'; \ No newline at end of file diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/embeddable_single_metric_viewer_container.tsx b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/embeddable_single_metric_viewer_container.tsx new file mode 100644 index 0000000000000..88c120c9747e1 --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/embeddable_single_metric_viewer_container.tsx @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; +import { EuiResizeObserver } from '@elastic/eui'; +import { Observable } from 'rxjs'; +import { throttle } from 'lodash'; +import { MlJob } from '@elastic/elasticsearch/lib/api/types'; +import usePrevious from 'react-use/lib/usePrevious'; +import { useToastNotificationService } from '../../application/services/toast_notification_service'; +import { useEmbeddableExecutionContext } from '../common/use_embeddable_execution_context'; +import { useSingleMetricViewerInputResolver } from './use_single_metric_viewer_input_resolver'; +import type { ISingleMetricViewerEmbeddable } from './single_metric_viewer_embeddable'; +import type { + SingleMetricViewerEmbeddableInput, + AnomalyChartsEmbeddableOutput, + SingleMetricViewerEmbeddableServices, +} from '..'; +import { ANOMALY_SINGLE_METRIC_VIEWER_EMBEDDABLE_TYPE } from '..'; +import { TimeSeriesExplorerEmbeddableChart } from '../../application/timeseriesexplorer/timeseriesexplorer_embeddable_chart'; +import { APP_STATE_ACTION } from '../../application/timeseriesexplorer/timeseriesexplorer_constants'; +import { useTimeSeriesExplorerService } from '../../application/util/time_series_explorer_service'; +import './_index.scss'; + +const RESIZE_THROTTLE_TIME_MS = 500; + +interface AppStateZoom { + from?: string; + to?: string; +} + +export interface EmbeddableSingleMetricViewerContainerProps { + id: string; + embeddableContext: InstanceType; + embeddableInput: Observable; + services: SingleMetricViewerEmbeddableServices; + refresh: Observable; + onInputChange: (input: Partial) => void; + onOutputChange: (output: Partial) => void; + onRenderComplete: () => void; + onLoading: () => void; + onError: (error: Error) => void; +} + +export const EmbeddableSingleMetricViewerContainer: FC< + EmbeddableSingleMetricViewerContainerProps +> = ({ + id, + embeddableContext, + embeddableInput, + services, + refresh, + onInputChange, + onOutputChange, + onRenderComplete, + onError, + onLoading, +}) => { + useEmbeddableExecutionContext( + services[0].executionContext, + embeddableInput, + ANOMALY_SINGLE_METRIC_VIEWER_EMBEDDABLE_TYPE, + id + ); + const [chartWidth, setChartWidth] = useState(0); + const [zoom, setZoom] = useState(); + const [selectedForecastId, setSelectedForecastId] = useState(); + const [detectorIndex, setDetectorIndex] = useState(0); + const [selectedJob, setSelectedJob] = useState(); + const [autoZoomDuration, setAutoZoomDuration] = useState(); + + const { mlApiServices } = services[2]; + const { data, bounds, lastRefresh } = useSingleMetricViewerInputResolver( + embeddableInput, + refresh, + services[1].data.query.timefilter.timefilter, + onRenderComplete + ); + const selectedJobId = data?.jobIds[0]; + const previousRefresh = usePrevious(lastRefresh ?? 0); + const mlTimeSeriesExplorer = useTimeSeriesExplorerService(); + + // Holds the container height for previously fetched data + const containerHeightRef = useRef(); + const toastNotificationService = useToastNotificationService(); + + useEffect( + function setUpSelectedJob() { + async function fetchSelectedJob() { + if (mlApiServices && selectedJobId !== undefined) { + const { jobs } = await mlApiServices.getJobs({ jobId: selectedJobId }); + const job = jobs[0]; + setSelectedJob(job); + } + } + fetchSelectedJob(); + }, + [selectedJobId, mlApiServices] + ); + + useEffect( + function setUpAutoZoom() { + let zoomDuration: number | undefined; + if (selectedJobId !== undefined && selectedJob !== undefined) { + zoomDuration = mlTimeSeriesExplorer.getAutoZoomDuration(selectedJob); + setAutoZoomDuration(zoomDuration); + } + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [selectedJobId, selectedJob?.job_id, mlTimeSeriesExplorer] + ); + + // eslint-disable-next-line react-hooks/exhaustive-deps + const resizeHandler = useCallback( + throttle((e: { width: number; height: number }) => { + // Keep previous container height so it doesn't change the page layout + containerHeightRef.current = e.height; + + if (Math.abs(chartWidth - e.width) > 20) { + setChartWidth(e.width); + } + }, RESIZE_THROTTLE_TIME_MS), + [chartWidth] + ); + + const appStateHandler = useCallback( + (action: string, payload?: any) => { + /** + * Empty zoom indicates that chart hasn't been rendered yet, + * hence any updates prior that should replace the URL state. + */ + + switch (action) { + case APP_STATE_ACTION.SET_DETECTOR_INDEX: + setDetectorIndex(payload); + break; + + case APP_STATE_ACTION.SET_FORECAST_ID: + setSelectedForecastId(payload); + setZoom(undefined); + break; + + case APP_STATE_ACTION.SET_ZOOM: + setZoom(payload); + break; + + case APP_STATE_ACTION.UNSET_ZOOM: + setZoom(undefined); + break; + } + }, + + [setZoom, setDetectorIndex, setSelectedForecastId] + ); + + const containerPadding = 10; + + return ( + + {(resizeRef) => ( +

+ {data !== undefined && autoZoomDuration !== undefined && ( + + )} +
+ )} + + ); +}; + +// required for dynamic import using React.lazy() +// eslint-disable-next-line import/no-default-export +export default EmbeddableSingleMetricViewerContainer; diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/embeddable_single_metric_viewer_container_lazy.tsx b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/embeddable_single_metric_viewer_container_lazy.tsx new file mode 100644 index 0000000000000..0a69aaf2c2deb --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/embeddable_single_metric_viewer_container_lazy.tsx @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +export const EmbeddableSingleMetricViewerContainer = React.lazy( + () => import('./embeddable_single_metric_viewer_container') +); diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/index.ts b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/index.ts new file mode 100644 index 0000000000000..9afdbe3d1298c --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { SingleMetricViewerEmbeddableFactory } from './single_metric_viewer_embeddable_factory'; diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable.tsx b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable.tsx new file mode 100644 index 0000000000000..82a1b5abc8b63 --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable.tsx @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { Suspense } from 'react'; +import ReactDOM from 'react-dom'; +import { pick } from 'lodash'; + +import { Embeddable } from '@kbn/embeddable-plugin/public'; + +import { CoreStart } from '@kbn/core/public'; +import { i18n } from '@kbn/i18n'; +import { Subject } from 'rxjs'; +import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { IContainer } from '@kbn/embeddable-plugin/public'; +import { DatePickerContextProvider, type DatePickerDependencies } from '@kbn/ml-date-picker'; +import { UI_SETTINGS } from '@kbn/data-plugin/common'; +import { EmbeddableSingleMetricViewerContainer } from './embeddable_single_metric_viewer_container_lazy'; +import type { JobId } from '../../../common/types/anomaly_detection_jobs'; +import type { MlDependencies } from '../../application/app'; +import { + ANOMALY_SINGLE_METRIC_VIEWER_EMBEDDABLE_TYPE, + SingleMetricViewerEmbeddableInput, + AnomalyChartsEmbeddableOutput, + SingleMetricViewerServices, +} from '..'; +import { EmbeddableLoading } from '../common/components/embeddable_loading_fallback'; + +export const getDefaultSingleMetricViewerPanelTitle = (jobIds: JobId[]) => + i18n.translate('xpack.ml.singleMetricViewerEmbeddable.title', { + defaultMessage: 'ML single metric viewer chart for {jobIds}', + values: { jobIds: jobIds.join(', ') }, + }); + +export type ISingleMetricViewerEmbeddable = typeof SingleMetricViewerEmbeddable; + +export class SingleMetricViewerEmbeddable extends Embeddable< + SingleMetricViewerEmbeddableInput, + AnomalyChartsEmbeddableOutput +> { + private node?: HTMLElement; + private reload$ = new Subject(); + public readonly type: string = ANOMALY_SINGLE_METRIC_VIEWER_EMBEDDABLE_TYPE; + + constructor( + initialInput: SingleMetricViewerEmbeddableInput, + public services: [CoreStart, MlDependencies, SingleMetricViewerServices], + parent?: IContainer + ) { + super(initialInput, {} as AnomalyChartsEmbeddableOutput, parent); + } + + public onLoading() { + this.renderComplete.dispatchInProgress(); + this.updateOutput({ loading: true, error: undefined }); + } + + public onError(error: Error) { + this.renderComplete.dispatchError(); + this.updateOutput({ loading: false, error: { name: error.name, message: error.message } }); + } + + public onRenderComplete() { + this.renderComplete.dispatchComplete(); + this.updateOutput({ loading: false, error: undefined }); + } + + public render(node: HTMLElement) { + super.render(node); + this.node = node; + + // required for the export feature to work + this.node.setAttribute('data-shared-item', ''); + + const I18nContext = this.services[0].i18n.Context; + const theme$ = this.services[0].theme.theme$; + + const datePickerDeps: DatePickerDependencies = { + ...pick(this.services[0], ['http', 'notifications', 'theme', 'uiSettings', 'i18n']), + data: this.services[1].data, + uiSettingsKeys: UI_SETTINGS, + showFrozenDataTierChoice: false, + }; + + ReactDOM.render( + + + + + }> + + + + + + , + node + ); + } + + public destroy() { + super.destroy(); + if (this.node) { + ReactDOM.unmountComponentAtNode(this.node); + } + } + + public reload() { + this.reload$.next(); + } + + public supportedTriggers() { + return []; + } +} diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable_factory.ts b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable_factory.ts new file mode 100644 index 0000000000000..06b2f9b024bfa --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable_factory.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +import type { StartServicesAccessor } from '@kbn/core/public'; +import type { EmbeddableFactoryDefinition, IContainer } from '@kbn/embeddable-plugin/public'; + +import { PLUGIN_ICON, PLUGIN_ID, ML_APP_NAME } from '../../../common/constants/app'; +import { + ANOMALY_SINGLE_METRIC_VIEWER_EMBEDDABLE_TYPE, + SingleMetricViewerEmbeddableInput, + SingleMetricViewerEmbeddableServices, +} from '..'; +import type { MlPluginStart, MlStartDependencies } from '../../plugin'; +import type { MlDependencies } from '../../application/app'; +import { HttpService } from '../../application/services/http_service'; +import { AnomalyExplorerChartsService } from '../../application/services/anomaly_explorer_charts_service'; + +export class SingleMetricViewerEmbeddableFactory + implements EmbeddableFactoryDefinition +{ + public readonly type = ANOMALY_SINGLE_METRIC_VIEWER_EMBEDDABLE_TYPE; + + public readonly grouping = [ + { + id: PLUGIN_ID, + getDisplayName: () => ML_APP_NAME, + getIconType: () => PLUGIN_ICON, + }, + ]; + + constructor( + private getStartServices: StartServicesAccessor + ) {} + + public async isEditable() { + return true; + } + + public getDisplayName() { + return i18n.translate('xpack.ml.components.mlSingleMetricViewerEmbeddable.displayName', { + defaultMessage: 'Single metric viewer', + }); + } + + public getDescription() { + return i18n.translate('xpack.ml.components.mlSingleMetricViewerEmbeddable.description', { + defaultMessage: 'View anomaly detection single metric results in a chart.', + }); + } + + public async getExplicitInput(): Promise> { + const [coreStart, pluginStart, singleMetricServices] = await this.getServices(); + + try { + const { resolveEmbeddableSingleMetricViewerUserInput } = await import( + './single_metric_viewer_setup_flyout' + ); + return await resolveEmbeddableSingleMetricViewerUserInput( + coreStart, + pluginStart, + singleMetricServices + ); + } catch (e) { + return Promise.reject(); + } + } + + private async getServices(): Promise { + const [ + [coreStart, pluginsStart], + { AnomalyDetectorService }, + { fieldFormatServiceFactory }, + { indexServiceFactory }, + { mlApiServicesProvider }, + { mlResultsServiceProvider }, + { timeSeriesSearchServiceFactory }, + ] = await Promise.all([ + await this.getStartServices(), + await import('../../application/services/anomaly_detector_service'), + await import('../../application/services/field_format_service_factory'), + await import('../../application/util/index_service'), + await import('../../application/services/ml_api_service'), + await import('../../application/services/results_service'), + await import( + '../../application/timeseriesexplorer/timeseriesexplorer_utils/time_series_search_service' + ), + ]); + + const httpService = new HttpService(coreStart.http); + const anomalyDetectorService = new AnomalyDetectorService(httpService); + const mlApiServices = mlApiServicesProvider(httpService); + const mlResultsService = mlResultsServiceProvider(mlApiServices); + const mlIndexUtils = indexServiceFactory(pluginsStart.data.dataViews); + const mlTimeSeriesSearchService = timeSeriesSearchServiceFactory( + mlResultsService, + mlApiServices + ); + const mlFieldFormatService = fieldFormatServiceFactory(mlApiServices, mlIndexUtils); + + const anomalyExplorerService = new AnomalyExplorerChartsService( + pluginsStart.data.query.timefilter.timefilter, + mlApiServices, + mlResultsService + ); + + return [ + coreStart, + pluginsStart as MlDependencies, + { + anomalyDetectorService, + anomalyExplorerService, + mlResultsService, + mlApiServices, + mlTimeSeriesSearchService, + mlFieldFormatService, + }, + ]; + } + + public async create(initialInput: SingleMetricViewerEmbeddableInput, parent?: IContainer) { + const services = await this.getServices(); + const { SingleMetricViewerEmbeddable } = await import('./single_metric_viewer_embeddable'); + return new SingleMetricViewerEmbeddable(initialInput, services, parent); + } +} diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_initializer.tsx b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_initializer.tsx new file mode 100644 index 0000000000000..89af056068063 --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_initializer.tsx @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useState } from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiForm, + EuiFormRow, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiFieldText, + EuiModal, + EuiSpacer, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { MlJob } from '@elastic/elasticsearch/lib/api/types'; +import type { SingleMetricViewerServices } from '..'; +import { TimeRangeBounds } from '../../application/util/time_buckets'; +import { SeriesControls } from '../../application/timeseriesexplorer/components/series_controls'; +import { + APP_STATE_ACTION, + type TimeseriesexplorerActionType, +} from '../../application/timeseriesexplorer/timeseriesexplorer_constants'; + +export interface SingleMetricViewerInitializerProps { + bounds: TimeRangeBounds; + defaultTitle: string; + initialInput?: SingleMetricViewerServices; + job: MlJob; + onCreate: (props: { + panelTitle: string; + functionDescription?: string; + selectedDetectorIndex: number; + selectedEntities: any; + }) => void; + onCancel: () => void; +} + +export const SingleMetricViewerInitializer: FC = ({ + bounds, + defaultTitle, + initialInput, + job, + onCreate, + onCancel, +}) => { + const [panelTitle, setPanelTitle] = useState(defaultTitle); + const [functionDescription, setFunctionDescription] = useState(); + const [selectedDetectorIndex, setSelectedDetectorIndex] = useState(0); + const [selectedEntities, setSelectedEntities] = useState(); + + const isPanelTitleValid = panelTitle.length > 0; + + const handleStateUpdate = (action: TimeseriesexplorerActionType, payload: any) => { + switch (action) { + case APP_STATE_ACTION.SET_ENTITIES: + setSelectedEntities(payload); + break; + case APP_STATE_ACTION.SET_FUNCTION_DESCRIPTION: + setFunctionDescription(payload); + break; + case APP_STATE_ACTION.SET_DETECTOR_INDEX: + setSelectedDetectorIndex(payload); + break; + default: + break; + } + }; + + return ( + + + + + + + + + + + } + isInvalid={!isPanelTitleValid} + > + setPanelTitle(e.target.value)} + isInvalid={!isPanelTitleValid} + /> + + + + + + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_setup_flyout.tsx b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_setup_flyout.tsx new file mode 100644 index 0000000000000..e9822c01f865a --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_setup_flyout.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import type { CoreStart } from '@kbn/core/public'; +import { toMountPoint } from '@kbn/react-kibana-mount'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { getDefaultSingleMetricViewerPanelTitle } from './single_metric_viewer_embeddable'; +import type { SingleMetricViewerEmbeddableInput, SingleMetricViewerServices } from '..'; +import { resolveJobSelection } from '../common/resolve_job_selection'; +import { SingleMetricViewerInitializer } from './single_metric_viewer_initializer'; +import type { MlStartDependencies } from '../../plugin'; + +export async function resolveEmbeddableSingleMetricViewerUserInput( + coreStart: CoreStart, + pluginStart: MlStartDependencies, + input: SingleMetricViewerServices +): Promise> { + const { overlays, theme, i18n } = coreStart; + const { mlApiServices } = input; + const timefilter = pluginStart.data.query.timefilter.timefilter; + + return new Promise(async (resolve, reject) => { + try { + const { jobIds } = await resolveJobSelection(coreStart, undefined, true); + const title = getDefaultSingleMetricViewerPanelTitle(jobIds); + const { jobs } = await mlApiServices.getJobs({ jobId: jobIds.join(',') }); + + const modalSession = overlays.openModal( + toMountPoint( + + { + modalSession.close(); + resolve({ + jobIds, + title: panelTitle, + functionDescription, + panelTitle, + selectedDetectorIndex, + selectedEntities, + }); + }} + onCancel={() => { + modalSession.close(); + reject(); + }} + /> + , + { theme, i18n } + ) + ); + } catch (error) { + reject(error); + } + }); +} diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/use_single_metric_viewer_input_resolver.ts b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/use_single_metric_viewer_input_resolver.ts new file mode 100644 index 0000000000000..c9f9d57fd7803 --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/use_single_metric_viewer_input_resolver.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useEffect, useState } from 'react'; +import { combineLatest, Observable } from 'rxjs'; +import { startWith } from 'rxjs/operators'; +import { TimefilterContract } from '@kbn/data-plugin/public'; +import { SingleMetricViewerEmbeddableInput } from '..'; +import type { TimeRangeBounds } from '../../application/util/time_buckets'; + +export function useSingleMetricViewerInputResolver( + embeddableInput: Observable, + refresh: Observable, + timefilter: TimefilterContract, + onRenderComplete: () => void +) { + const [data, setData] = useState(); + const [bounds, setBounds] = useState(); + const [lastRefresh, setLastRefresh] = useState(); + + useEffect(function subscribeToEmbeddableInput() { + const subscription = combineLatest([embeddableInput, refresh.pipe(startWith(null))]).subscribe( + (input) => { + if (input !== undefined) { + setData(input[0]); + if (timefilter !== undefined) { + const { timeRange } = input[0]; + const currentBounds = timefilter.calculateBounds(timeRange); + setBounds(currentBounds); + setLastRefresh(Date.now()); + } + onRenderComplete(); + } + } + ); + + return () => subscription.unsubscribe(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return { data, bounds, lastRefresh }; +} diff --git a/x-pack/plugins/ml/public/embeddables/types.ts b/x-pack/plugins/ml/public/embeddables/types.ts index 48a7b5d43a5ac..56a33f488d534 100644 --- a/x-pack/plugins/ml/public/embeddables/types.ts +++ b/x-pack/plugins/ml/public/embeddables/types.ts @@ -27,6 +27,9 @@ import { MlEmbeddableTypes, } from './constants'; import { MlResultsService } from '../application/services/results_service'; +import type { MlApiServices } from '../application/services/ml_api_service'; +import type { MlFieldFormatService } from '../application/services/field_format_service'; +import type { MlTimeSeriesSeachService } from '../application/timeseriesexplorer/timeseriesexplorer_utils/time_series_search_service'; export interface AnomalySwimlaneEmbeddableCustomInput { jobIds: JobId[]; @@ -100,13 +103,45 @@ export interface AnomalyChartsEmbeddableCustomInput { export type AnomalyChartsEmbeddableInput = EmbeddableInput & AnomalyChartsEmbeddableCustomInput; +export interface SingleMetricViewerEmbeddableCustomInput { + jobIds: JobId[]; + title: string; + functionDescription?: string; + panelTitle: string; + selectedDetectorIndex: number; + selectedEntities: MlEntityField[]; + // Embeddable inputs which are not included in the default interface + filters: Filter[]; + query: Query; + refreshConfig: RefreshInterval; + timeRange: TimeRange; +} + +export type SingleMetricViewerEmbeddableInput = EmbeddableInput & + SingleMetricViewerEmbeddableCustomInput; + export interface AnomalyChartsServices { anomalyDetectorService: AnomalyDetectorService; anomalyExplorerService: AnomalyExplorerChartsService; mlResultsService: MlResultsService; + mlApiServices?: MlApiServices; +} + +export interface SingleMetricViewerServices { + anomalyExplorerService: AnomalyExplorerChartsService; + anomalyDetectorService: AnomalyDetectorService; + mlApiServices: MlApiServices; + mlFieldFormatService: MlFieldFormatService; + mlResultsService: MlResultsService; + mlTimeSeriesSearchService?: MlTimeSeriesSeachService; } export type AnomalyChartsEmbeddableServices = [CoreStart, MlDependencies, AnomalyChartsServices]; +export type SingleMetricViewerEmbeddableServices = [ + CoreStart, + MlDependencies, + SingleMetricViewerServices +]; export interface AnomalyChartsCustomOutput { entityFields?: MlEntityField[]; severity?: number; diff --git a/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts b/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts index ee0fae1f91ed1..909a823286cc6 100644 --- a/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts +++ b/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts @@ -9,7 +9,7 @@ import { decodeOrThrow, jsonRt } from '@kbn/io-ts-utils'; import type { Serializable } from '@kbn/utility-types'; import dedent from 'dedent'; import * as t from 'io-ts'; -import { last, omit } from 'lodash'; +import { compact, last, omit } from 'lodash'; import { lastValueFrom } from 'rxjs'; import { FunctionRegistrationParameters } from '.'; import { MessageRole, type Message } from '../../common/types'; @@ -88,12 +88,17 @@ export function registerRecallFunction({ messages.filter((message) => message.message.role === MessageRole.User) ); + const nonEmptyQueries = compact(queries); + + const queriesOrUserPrompt = nonEmptyQueries.length + ? nonEmptyQueries + : compact([userMessage?.message.content]); + const suggestions = await retrieveSuggestions({ userMessage, client, - signal, categories, - queries, + queries: queriesOrUserPrompt, }); if (suggestions.length === 0) { @@ -104,9 +109,8 @@ export function registerRecallFunction({ const relevantDocuments = await scoreSuggestions({ suggestions, - systemMessage, - userMessage, - queries, + queries: queriesOrUserPrompt, + messages, client, connectorId, signal, @@ -121,25 +125,17 @@ export function registerRecallFunction({ } async function retrieveSuggestions({ - userMessage, queries, client, categories, - signal, }: { userMessage?: Message; queries: string[]; client: ObservabilityAIAssistantClient; categories: Array<'apm' | 'lens'>; - signal: AbortSignal; }) { - const queriesWithUserPrompt = - userMessage && userMessage.message.content - ? [userMessage.message.content, ...queries] - : queries; - const recallResponse = await client.recall({ - queries: queriesWithUserPrompt, + queries, categories, }); @@ -156,18 +152,12 @@ const scoreFunctionRequestRt = t.type({ }); const scoreFunctionArgumentsRt = t.type({ - scores: t.array( - t.type({ - id: t.string, - score: t.number, - }) - ), + scores: t.string, }); async function scoreSuggestions({ suggestions, - systemMessage, - userMessage, + messages, queries, client, connectorId, @@ -175,35 +165,31 @@ async function scoreSuggestions({ resources, }: { suggestions: Awaited>; - systemMessage: Message; - userMessage?: Message; + messages: Message[]; queries: string[]; client: ObservabilityAIAssistantClient; connectorId: string; signal: AbortSignal; resources: RespondFunctionResources; }) { - resources.logger.debug(`Suggestions: ${JSON.stringify(suggestions, null, 2)}`); + const indexedSuggestions = suggestions.map((suggestion, index) => ({ ...suggestion, id: index })); - const systemMessageExtension = - dedent(`You have the function called score available to help you inform the user about how relevant you think a given document is to the conversation. - Please give a score between 1 and 7, fractions are allowed. - A higher score means it is more relevant.`); - const extendedSystemMessage = { - ...systemMessage, - message: { - ...systemMessage.message, - content: `${systemMessage.message.content}\n\n${systemMessageExtension}`, - }, - }; + const newUserMessageContent = + dedent(`Given the following question, score the documents that are relevant to the question. on a scale from 0 to 7, + 0 being completely relevant, and 7 being extremely relevant. Information is relevant to the question if it helps in + answering the question. Judge it according to the following criteria: - const userMessageOrQueries = - userMessage && userMessage.message.content ? userMessage.message.content : queries.join(','); + - The document is relevant to the question, and the rest of the conversation + - The document has information relevant to the question that is not mentioned, + or more detailed than what is available in the conversation + - The document has a high amount of information relevant to the question compared to other documents + - The document contains new information not mentioned before in the conversation - const newUserMessageContent = - dedent(`Given the question "${userMessageOrQueries}", can you give me a score for how relevant the following documents are? + Question: + ${queries.join('\n')} - ${JSON.stringify(suggestions, null, 2)}`); + Documents: + ${JSON.stringify(indexedSuggestions, null, 2)}`); const newUserMessage: Message = { '@timestamp': new Date().toISOString(), @@ -222,22 +208,13 @@ async function scoreSuggestions({ additionalProperties: false, properties: { scores: { - description: 'The document IDs and their scores', - type: 'array', - items: { - type: 'object', - additionalProperties: false, - properties: { - id: { - description: 'The ID of the document', - type: 'string', - }, - score: { - description: 'The score for the document', - type: 'number', - }, - }, - }, + description: `The document IDs and their scores, as CSV. Example: + + my_id,7 + my_other_id,3 + my_third_id,4 + `, + type: 'string', }, }, required: ['score'], @@ -249,7 +226,7 @@ async function scoreSuggestions({ ( await client.chat('score_suggestions', { connectorId, - messages: [extendedSystemMessage, newUserMessage], + messages: [...messages.slice(0, -1), newUserMessage], functions: [scoreFunction], functionCall: 'score', signal, @@ -257,11 +234,18 @@ async function scoreSuggestions({ ).pipe(concatenateChatCompletionChunks()) ); const scoreFunctionRequest = decodeOrThrow(scoreFunctionRequestRt)(response); - const { scores } = decodeOrThrow(jsonRt.pipe(scoreFunctionArgumentsRt))( + const { scores: scoresAsString } = decodeOrThrow(jsonRt.pipe(scoreFunctionArgumentsRt))( scoreFunctionRequest.message.function_call.arguments ); - resources.logger.debug(`Scores: ${JSON.stringify(scores, null, 2)}`); + const scores = scoresAsString.split('\n').map((line) => { + const [index, score] = line + .split(',') + .map((value) => value.trim()) + .map(Number); + + return { id: suggestions[index].id, score }; + }); if (scores.length === 0) { return []; diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts index f3ab3e917979b..afd34aa8ea966 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts @@ -14,7 +14,15 @@ import apm from 'elastic-apm-node'; import { decode, encode } from 'gpt-tokenizer'; import { compact, isEmpty, last, merge, noop, omit, pick, take } from 'lodash'; import type OpenAI from 'openai'; -import { filter, isObservable, lastValueFrom, Observable, shareReplay, toArray } from 'rxjs'; +import { + filter, + firstValueFrom, + isObservable, + lastValueFrom, + Observable, + shareReplay, + toArray, +} from 'rxjs'; import { Readable } from 'stream'; import { v4 } from 'uuid'; import { @@ -455,6 +463,8 @@ export class ObservabilityAIAssistantClient { ): Promise> => { const span = apm.startSpan(`chat ${name}`); + const spanId = (span?.ids['span.id'] || '').substring(0, 6); + const messagesForOpenAI: Array< Omit & { role: MessageRole; @@ -490,6 +500,8 @@ export class ObservabilityAIAssistantClient { this.dependencies.logger.debug(`Sending conversation to connector`); this.dependencies.logger.trace(JSON.stringify(request, null, 2)); + const now = performance.now(); + const executeResult = await this.dependencies.actionsClient.execute({ actionId: connectorId, params: { @@ -501,7 +513,11 @@ export class ObservabilityAIAssistantClient { }, }); - this.dependencies.logger.debug(`Received action client response: ${executeResult.status}`); + this.dependencies.logger.debug( + `Received action client response: ${executeResult.status} (took: ${Math.round( + performance.now() - now + )}ms)${spanId ? ` (${spanId})` : ''}` + ); if (executeResult.status === 'error' && executeResult?.serviceMessage) { const tokenLimitRegex = @@ -524,20 +540,34 @@ export class ObservabilityAIAssistantClient { const observable = streamIntoObservable(response).pipe(processOpenAiStream(), shareReplay()); - if (span) { - lastValueFrom(observable) - .then( - () => { - span.setOutcome('success'); - }, - () => { - span.setOutcome('failure'); - } - ) - .finally(() => { - span.end(); - }); - } + firstValueFrom(observable) + .catch(noop) + .finally(() => { + this.dependencies.logger.debug( + `Received first value after ${Math.round(performance.now() - now)}ms${ + spanId ? ` (${spanId})` : '' + }` + ); + }); + + lastValueFrom(observable) + .then( + () => { + span?.setOutcome('success'); + }, + () => { + span?.setOutcome('failure'); + } + ) + .finally(() => { + this.dependencies.logger.debug( + `Completed response in ${Math.round(performance.now() - now)}ms${ + spanId ? ` (${spanId})` : '' + }` + ); + + span?.end(); + }); return observable; }; diff --git a/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts b/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts index 62772fa029e66..cfa004b7ce6b2 100644 --- a/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts +++ b/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts @@ -7,6 +7,7 @@ import type { ReactNode } from 'react'; import { useCallback, useMemo } from 'react'; import type { TimelineEventsDetailsItem } from '@kbn/timelines-plugin/common'; +import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; import { getSentinelOneAgentId } from '../../../common/utils/sentinelone_alert_check'; import type { ThirdPartyAgentInfo } from '../../../../common/types'; import type { ResponseActionAgentType } from '../../../../common/endpoint/service/response_actions/constants'; @@ -74,17 +75,22 @@ export const useResponderActionData = ({ tooltip: ReactNode; } => { const isEndpointHost = agentType === 'endpoint'; + const isSentinelOneHost = agentType === 'sentinel_one'; const showResponseActionsConsole = useWithShowResponder(); + const isSentinelOneV1Enabled = useIsExperimentalFeatureEnabled( + 'responseActionsSentinelOneV1Enabled' + ); const { data: hostInfo, isFetching, error, - } = useGetEndpointDetails(endpointId, { enabled: Boolean(endpointId) }); + } = useGetEndpointDetails(endpointId, { enabled: Boolean(endpointId && isEndpointHost) }); const [isDisabled, tooltip]: [disabled: boolean, tooltip: ReactNode] = useMemo(() => { + // v8.13 disabled for third-party agent alerts if the feature flag is not enabled if (!isEndpointHost) { - return [false, undefined]; + return [isSentinelOneHost ? !isSentinelOneV1Enabled : true, undefined]; } // Still loading host info @@ -114,7 +120,14 @@ export const useResponderActionData = ({ } return [false, undefined]; - }, [isEndpointHost, isFetching, error, hostInfo?.host_status]); + }, [ + isEndpointHost, + isSentinelOneHost, + isSentinelOneV1Enabled, + isFetching, + error, + hostInfo?.host_status, + ]); const handleResponseActionsClick = useCallback(() => { if (!isEndpointHost) { diff --git a/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx b/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx index 7e3aa104f5a60..fd9a63f5547e0 100644 --- a/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx +++ b/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx @@ -116,6 +116,7 @@ export const EsqlQueryExpression: React.FC< from: new Date(now - timeWindow).toISOString(), to: new Date(now).toISOString(), }, + undefined, // create a data view with the timefield to pass into the query new DataView({ spec: { timeFieldName: timeField }, @@ -219,7 +220,7 @@ export const EsqlQueryExpression: React.FC< }, 1000)} expandCodeEditor={() => true} isCodeEditorExpanded={true} - onTextLangQuerySubmit={() => {}} + onTextLangQuerySubmit={async () => {}} detectTimestamp={detectTimestamp} hideMinimizeButton={true} hideRunQueryText={true} diff --git a/x-pack/test/api_integration/apis/maps/maps_telemetry.ts b/x-pack/test/api_integration/apis/maps/maps_telemetry.ts index 92ae21c7c09c0..8f5c9ae95f8af 100644 --- a/x-pack/test/api_integration/apis/maps/maps_telemetry.ts +++ b/x-pack/test/api_integration/apis/maps/maps_telemetry.ts @@ -60,7 +60,7 @@ export default function ({ getService }: FtrProviderContext) { es_point_to_point: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 }, es_top_hits: { min: 1, max: 1, total: 2, avg: 0.07142857142857142 }, es_agg_heatmap: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 }, - esql: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 }, + esql: { min: 1, max: 1, total: 2, avg: 0.07142857142857142 }, kbn_tms_raster: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 }, ems_basemap: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 }, ems_region: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 }, @@ -81,8 +81,8 @@ export default function ({ getService }: FtrProviderContext) { min: 0, }, dataSourcesCount: { - avg: 1.1785714285714286, - max: 6, + avg: 1.2142857142857142, + max: 7, min: 1, }, emsVectorLayersCount: { @@ -104,8 +104,8 @@ export default function ({ getService }: FtrProviderContext) { min: 1, }, GEOJSON_VECTOR: { - avg: 0.8214285714285714, - max: 5, + avg: 0.8571428571428571, + max: 6, min: 1, }, HEATMAP: { @@ -125,8 +125,8 @@ export default function ({ getService }: FtrProviderContext) { }, }, layersCount: { - avg: 1.2142857142857142, - max: 7, + avg: 1.25, + max: 8, min: 1, }, }, diff --git a/x-pack/test/functional/apps/index_management/index_template_wizard.ts b/x-pack/test/functional/apps/index_management/index_template_wizard.ts index 8776c6de06e40..2a16849f2f5de 100644 --- a/x-pack/test/functional/apps/index_management/index_template_wizard.ts +++ b/x-pack/test/functional/apps/index_management/index_template_wizard.ts @@ -117,13 +117,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await testSubjects.setValue('indexPatternsField', 'test-index-pattern'); // Go to Mappings step - await pageObjects.indexManagement.clickNextButton(); - expect(await testSubjects.getVisibleText('stepTitle')).to.be( - 'Component templates (optional)' - ); - await pageObjects.indexManagement.clickNextButton(); - expect(await testSubjects.getVisibleText('stepTitle')).to.be('Index settings (optional)'); - await pageObjects.indexManagement.clickNextButton(); + await testSubjects.click('formWizardStep-3'); expect(await testSubjects.getVisibleText('stepTitle')).to.be('Mappings (optional)'); }); diff --git a/x-pack/test/functional/apps/maps/group4/layer_errors.js b/x-pack/test/functional/apps/maps/group4/layer_errors.js index e47c0e582c8f4..9f8a570a46d96 100644 --- a/x-pack/test/functional/apps/maps/group4/layer_errors.js +++ b/x-pack/test/functional/apps/maps/group4/layer_errors.js @@ -18,6 +18,20 @@ export default function ({ getPageObjects, getService }) { await PageObjects.maps.loadSavedMap('layer with errors'); }); + describe('Layer with invalid descriptor', () => { + const INVALID_LAYER_NAME = 'fff76ebb-57a6-4067-a373-1d191b9bd1a3'; + + it('should diplay error icon in legend', async () => { + await PageObjects.maps.hasErrorIconExistsOrFail(INVALID_LAYER_NAME); + }); + + it('should allow deletion of layer', async () => { + await PageObjects.maps.removeLayer(INVALID_LAYER_NAME); + const exists = await PageObjects.maps.doesLayerExist(INVALID_LAYER_NAME); + expect(exists).to.be(false); + }); + }); + describe('Layer with EsError', () => { after(async () => { await inspector.close(); diff --git a/x-pack/test/functional/fixtures/kbn_archiver/maps.json b/x-pack/test/functional/fixtures/kbn_archiver/maps.json index 74f27e360cfa1..1ce5bc3fd6aa1 100644 --- a/x-pack/test/functional/fixtures/kbn_archiver/maps.json +++ b/x-pack/test/functional/fixtures/kbn_archiver/maps.json @@ -747,7 +747,7 @@ "version": "WzU1LDFd", "attributes": { "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"KIBANA_TILEMAP\"},\"id\":\"ap0ys\",\"label\":\"Custom_TMS\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"TILE\",\"properties\":{},\"previousStyle\":null},\"type\":\"RASTER_TILE\"},{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"id\":\"idThatDoesNotExitForEMSTile\",\"lightModeDefault\":\"road_map\"},\"temporary\":false,\"id\":\"plw9l\",\"label\":\"EMS_tiles\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"TILE\",\"properties\":{},\"previousStyle\":null},\"type\":\"EMS_VECTOR_TILE\"},{\"sourceDescriptor\":{\"type\":\"EMS_FILE\",\"id\":\"idThatDoesNotExitForEMSFileSource\"},\"temporary\":false,\"id\":\"2gro0\",\"label\":\"EMS_vector_shapes\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#e6194b\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\"},{\"sourceDescriptor\":{\"type\":\"ES_GEO_GRID\",\"id\":\"f67fe707-95dd-46d6-89b8-82617b251b61\",\"geoField\":\"geo.coordinates\",\"requestType\":\"grid\",\"resolution\":\"COARSE\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_3_source_index_pattern\"},\"temporary\":false,\"id\":\"pl5qd\",\"label\":\"\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"Count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"color\":\"Blues\",\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"Count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"minSize\":4,\"maxSize\":32,\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\"},{\"sourceDescriptor\":{\"id\":\"a07072bb-3a92-4320-bd37-250ef6d04db7\",\"type\":\"ES_SEARCH\",\"geoField\":\"geo.coordinates\",\"limit\":2048,\"filterByMapBounds\":true,\"tooltipProperties\":[],\"applyGlobalQuery\":true,\"scalingType\":\"LIMIT\",\"indexPatternRefName\":\"layer_4_source_index_pattern\"},\"temporary\":false,\"id\":\"9bw8h\",\"label\":\"\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#e6194b\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\"},{\"id\":\"n1t6f\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"id\":\"62eca1fc-fe42-11e8-8eb2-f2801f1b9fd1\",\"type\":\"ES_SEARCH\",\"geoField\":\"geometry\",\"limit\":2048,\"filterByMapBounds\":false,\"showTooltip\":true,\"tooltipProperties\":[],\"applyGlobalQuery\":true,\"scalingType\":\"LIMIT\",\"indexPatternRefName\":\"layer_5_source_index_pattern\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"max(prop1) group by meta_for_geo_shapes*.shape_name\",\"name\":\"__kbnjoin__max_of_prop1__855ccb86-fe42-11e8-8eb2-f2801f1b9fd1\",\"origin\":\"join\"},\"color\":\"Blues\",\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"temporary\":true,\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\",\"joins\":[{\"leftField\":\"name\",\"right\":{\"id\":\"855ccb86-fe42-11e8-8eb2-f2801f1b9fd1\",\"indexPatternTitle\":\"meta_for_geo_shapes*\",\"term\":\"shape_name\",\"metrics\":[{\"type\":\"max\",\"field\":\"prop1\"}],\"applyGlobalQuery\":true,\"type\":\"ES_TERM_SOURCE\",\"indexPatternRefName\":\"layer_5_join_0_index_pattern\"}}]},{\"sourceDescriptor\":{\"geoField\":\"destination\",\"scalingType\":\"LIMIT\",\"id\":\"ed01aac3-c0be-491e-98c9-f1cb6e37f185\",\"type\":\"ES_SEARCH\",\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"filterByMapBounds\":true,\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"topHitsGroupByTimeseries\":false,\"topHitsSplitField\":\"\",\"topHitsSize\":1,\"indexPatternRefName\":\"layer_6_source_index_pattern\"},\"id\":\"1cfaa7fa-dc73-419d-b362-7238e2270a1c\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":0}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelZoomRange\":{\"options\":{\"useLayerZoomRange\":true,\"minZoom\":0,\"maxZoom\":24}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}},\"labelPosition\":{\"options\":{\"position\":\"CENTER\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"GEOJSON_VECTOR\",\"joins\":[],\"disableTooltips\":false}]", + "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"KIBANA_TILEMAP\"},\"id\":\"ap0ys\",\"label\":\"Custom_TMS\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"TILE\",\"properties\":{},\"previousStyle\":null},\"type\":\"RASTER_TILE\"},{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"id\":\"idThatDoesNotExitForEMSTile\",\"lightModeDefault\":\"road_map\"},\"temporary\":false,\"id\":\"plw9l\",\"label\":\"EMS_tiles\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"TILE\",\"properties\":{},\"previousStyle\":null},\"type\":\"EMS_VECTOR_TILE\"},{\"sourceDescriptor\":{\"type\":\"EMS_FILE\",\"id\":\"idThatDoesNotExitForEMSFileSource\"},\"temporary\":false,\"id\":\"2gro0\",\"label\":\"EMS_vector_shapes\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#e6194b\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\"},{\"sourceDescriptor\":{\"type\":\"ES_GEO_GRID\",\"id\":\"f67fe707-95dd-46d6-89b8-82617b251b61\",\"geoField\":\"geo.coordinates\",\"requestType\":\"grid\",\"resolution\":\"COARSE\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_3_source_index_pattern\"},\"temporary\":false,\"id\":\"pl5qd\",\"label\":\"\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"Count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"color\":\"Blues\",\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"Count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"minSize\":4,\"maxSize\":32,\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\"},{\"sourceDescriptor\":{\"id\":\"a07072bb-3a92-4320-bd37-250ef6d04db7\",\"type\":\"ES_SEARCH\",\"geoField\":\"geo.coordinates\",\"limit\":2048,\"filterByMapBounds\":true,\"tooltipProperties\":[],\"applyGlobalQuery\":true,\"scalingType\":\"LIMIT\",\"indexPatternRefName\":\"layer_4_source_index_pattern\"},\"temporary\":false,\"id\":\"9bw8h\",\"label\":\"\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#e6194b\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\"},{\"id\":\"n1t6f\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"id\":\"62eca1fc-fe42-11e8-8eb2-f2801f1b9fd1\",\"type\":\"ES_SEARCH\",\"geoField\":\"geometry\",\"limit\":2048,\"filterByMapBounds\":false,\"showTooltip\":true,\"tooltipProperties\":[],\"applyGlobalQuery\":true,\"scalingType\":\"LIMIT\",\"indexPatternRefName\":\"layer_5_source_index_pattern\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"max(prop1) group by meta_for_geo_shapes*.shape_name\",\"name\":\"__kbnjoin__max_of_prop1__855ccb86-fe42-11e8-8eb2-f2801f1b9fd1\",\"origin\":\"join\"},\"color\":\"Blues\",\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}}},\"temporary\":true,\"previousStyle\":null},\"type\":\"GEOJSON_VECTOR\",\"joins\":[{\"leftField\":\"name\",\"right\":{\"id\":\"855ccb86-fe42-11e8-8eb2-f2801f1b9fd1\",\"indexPatternTitle\":\"meta_for_geo_shapes*\",\"term\":\"shape_name\",\"metrics\":[{\"type\":\"max\",\"field\":\"prop1\"}],\"applyGlobalQuery\":true,\"type\":\"ES_TERM_SOURCE\",\"indexPatternRefName\":\"layer_5_join_0_index_pattern\"}}]},{\"sourceDescriptor\":{\"geoField\":\"destination\",\"scalingType\":\"LIMIT\",\"id\":\"ed01aac3-c0be-491e-98c9-f1cb6e37f185\",\"type\":\"ES_SEARCH\",\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"filterByMapBounds\":true,\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"topHitsGroupByTimeseries\":false,\"topHitsSplitField\":\"\",\"topHitsSize\":1,\"indexPatternRefName\":\"layer_6_source_index_pattern\"},\"id\":\"1cfaa7fa-dc73-419d-b362-7238e2270a1c\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":0}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelZoomRange\":{\"options\":{\"useLayerZoomRange\":true,\"minZoom\":0,\"maxZoom\":24}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}},\"labelPosition\":{\"options\":{\"position\":\"CENTER\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"GEOJSON_VECTOR\",\"joins\":[],\"disableTooltips\":false},{\"sourceDescriptor\":{\"id\":\"d4d6d4cf-58ee-4a0d-a792-532c0711fa2a\",\"type\":\"ESQL\"},\"id\":\"fff76ebb-57a6-4067-a373-1d191b9bd1a3\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#6092C0\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#4379aa\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelZoomRange\":{\"options\":{\"useLayerZoomRange\":true,\"minZoom\":0,\"maxZoom\":24}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}},\"labelPosition\":{\"options\":{\"position\":\"CENTER\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"GEOJSON_VECTOR\",\"joins\":[],\"disableTooltips\":false}]", "mapStateJSON": "{\"adHocDataViews\":[],\"zoom\":3.38,\"center\":{\"lon\":76.34937,\"lat\":-77.25604},\"timeFilters\":{\"from\":\"now-7d\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"index\":\"561253e0-f731-11e8-8487-11b9dd924f96\",\"type\":\"custom\",\"disabled\":false,\"negate\":false,\"alias\":\"connections shard failure\",\"key\":\"query\",\"value\":\"{\\\"error_query\\\":{\\\"indices\\\":[{\\\"error_type\\\":\\\"exception\\\",\\\"message\\\":\\\"simulated shard failure\\\",\\\"name\\\":\\\"connections\\\"}]}}\"},\"$state\":{\"store\":\"appState\"},\"query\":{\"error_query\":{\"indices\":[{\"error_type\":\"exception\",\"message\":\"simulated shard failure\",\"name\":\"connections\"}]}}}],\"settings\":{\"autoFitToDataBounds\":false,\"backgroundColor\":\"#ffffff\",\"customIcons\":[],\"disableInteractive\":false,\"disableTooltipControl\":false,\"hideToolbarOverlay\":false,\"hideLayerControl\":false,\"hideViewControl\":false,\"initialLocation\":\"LAST_SAVED_LOCATION\",\"fixedLocation\":{\"lat\":0,\"lon\":0,\"zoom\":2},\"browserLocation\":{\"zoom\":2},\"keydownScrollZoom\":false,\"maxZoom\":24,\"minZoom\":0,\"showScaleControl\":false,\"showSpatialFilters\":true,\"showTimesliderToggleButton\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}", "title": "layer with errors", "uiStateJSON": "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[\"1cfaa7fa-dc73-419d-b362-7238e2270a1c\"]}"