diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 2841dce49bb26..fcf4a82c0801c 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -372,7 +372,7 @@ steps: agents: queue: n2-8-spot key: storybooks - timeout_in_minutes: 60 + timeout_in_minutes: 80 retry: automatic: - exit_status: '-1' diff --git a/.buildkite/pipelines/pull_request/storybooks.yml b/.buildkite/pipelines/pull_request/storybooks.yml index 81d13b628e049..8f76879231de2 100644 --- a/.buildkite/pipelines/pull_request/storybooks.yml +++ b/.buildkite/pipelines/pull_request/storybooks.yml @@ -4,4 +4,4 @@ steps: agents: queue: n2-8-spot key: storybooks - timeout_in_minutes: 60 + timeout_in_minutes: 80 diff --git a/docs/management/connectors/pre-configured-connectors.asciidoc b/docs/management/connectors/pre-configured-connectors.asciidoc index b7293b6232190..c027220376cdf 100644 --- a/docs/management/connectors/pre-configured-connectors.asciidoc +++ b/docs/management/connectors/pre-configured-connectors.asciidoc @@ -148,7 +148,7 @@ xpack.actions.preconfigured: actionTypeId: .bedrock config: apiUrl: https://bedrock-runtime.us-east-1.amazonaws.com <1> - defaultModel: anthropic.claude-v2 <2> + defaultModel: anthropic.claude-v2:1 <2> secrets: accessKey: key-value <3> secret: secret-value <4> diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc index b7d7e8d344a32..2bfde478a494d 100644 --- a/docs/settings/alert-action-settings.asciidoc +++ b/docs/settings/alert-action-settings.asciidoc @@ -340,7 +340,7 @@ For a <>, specifies a string f The default model to use for requests, which varies by connector: + -- -* For an <>, current support is for the Anthropic Claude models. Defaults to `anthropic.claude-v2`. +* For an <>, current support is for the Anthropic Claude models. Defaults to `anthropic.claude-v2:1`. * For a <>, it is optional and applicable only when `xpack.actions.preconfigured..config.apiProvider` is `OpenAI`. -- diff --git a/package.json b/package.json index 05b09aa56196c..c11b0b349099f 100644 --- a/package.json +++ b/package.json @@ -881,6 +881,8 @@ "@reduxjs/toolkit": "1.9.7", "@slack/webhook": "^7.0.1", "@smithy/eventstream-codec": "^2.0.12", + "@smithy/eventstream-serde-node": "^2.1.1", + "@smithy/types": "^2.9.1", "@smithy/util-utf8": "^2.0.0", "@tanstack/react-query": "^4.29.12", "@tanstack/react-query-devtools": "^4.29.12", @@ -946,6 +948,7 @@ "diff": "^5.1.0", "elastic-apm-node": "^4.4.0", "email-addresses": "^5.0.0", + "eventsource-parser": "^1.1.1", "execa": "^5.1.1", "expiry-js": "0.1.7", "exponential-backoff": "^3.1.1", @@ -954,6 +957,7 @@ "fast-glob": "^3.3.2", "fflate": "^0.6.9", "file-saver": "^1.3.8", + "flat": "5", "fnv-plus": "^1.3.1", "font-awesome": "4.7.0", "formik": "^2.4.5", @@ -1380,11 +1384,13 @@ "@types/ejs": "^3.0.6", "@types/enzyme": "^3.10.12", "@types/eslint": "^8.44.2", + "@types/event-stream": "^4.0.5", "@types/express": "^4.17.13", "@types/extract-zip": "^1.6.2", "@types/faker": "^5.1.5", "@types/fetch-mock": "^7.3.1", "@types/file-saver": "^2.0.0", + "@types/flat": "^5.0.5", "@types/flot": "^0.0.31", "@types/fnv-plus": "^1.3.0", "@types/geojson": "^7946.0.10", diff --git a/packages/kbn-search-connectors/lib/index.ts b/packages/kbn-search-connectors/lib/index.ts index e0a1caea66422..e7269a0620b62 100644 --- a/packages/kbn-search-connectors/lib/index.ts +++ b/packages/kbn-search-connectors/lib/index.ts @@ -23,5 +23,6 @@ export * from './update_connector_configuration'; export * from './update_connector_index_name'; export * from './update_connector_name_and_description'; export * from './update_connector_scheduling'; +export * from './update_connector_secret'; export * from './update_connector_service_type'; export * from './update_connector_status'; diff --git a/packages/kbn-search-connectors/lib/update_connector_secret.test.ts b/packages/kbn-search-connectors/lib/update_connector_secret.test.ts new file mode 100644 index 0000000000000..80eb98a37babd --- /dev/null +++ b/packages/kbn-search-connectors/lib/update_connector_secret.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; + +import { updateConnectorSecret } from './update_connector_secret'; + +describe('updateConnectorSecret lib function', () => { + const mockClient = { + transport: { + request: jest.fn(), + }, + }; + + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + }); + + it('should update a connector secret', async () => { + mockClient.transport.request.mockImplementation(() => ({ + result: 'created', + })); + + await expect( + updateConnectorSecret(mockClient as unknown as ElasticsearchClient, 'my-secret', 'secret-id') + ).resolves.toEqual({ result: 'created' }); + expect(mockClient.transport.request).toHaveBeenCalledWith({ + method: 'PUT', + path: '/_connector/_secret/secret-id', + body: { + value: 'my-secret', + }, + }); + jest.useRealTimers(); + }); +}); diff --git a/packages/kbn-search-connectors/lib/update_connector_secret.ts b/packages/kbn-search-connectors/lib/update_connector_secret.ts new file mode 100644 index 0000000000000..516818b7e9b8d --- /dev/null +++ b/packages/kbn-search-connectors/lib/update_connector_secret.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { ConnectorsAPIUpdateResponse } from '../types/connectors_api'; + +export const updateConnectorSecret = async ( + client: ElasticsearchClient, + value: string, + secretId: string +) => { + return await client.transport.request({ + method: 'PUT', + path: `/_connector/_secret/${secretId}`, + body: { + value, + }, + }); +}; diff --git a/packages/kbn-test/jest-preset.js b/packages/kbn-test/jest-preset.js index df9ed4cab4f51..de4e6032ba52f 100644 --- a/packages/kbn-test/jest-preset.js +++ b/packages/kbn-test/jest-preset.js @@ -105,7 +105,7 @@ module.exports = { transformIgnorePatterns: [ // ignore all node_modules except monaco-editor, monaco-yaml and react-monaco-editor which requires babel transforms to handle dynamic import() // since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842) - '[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|monaco-languageserver-types|monaco-marker-data-provider|monaco-worker-manager|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color|langchain|langsmith|@cfworker|gpt-tokenizer))[/\\\\].+\\.js$', + '[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|monaco-languageserver-types|monaco-marker-data-provider|monaco-worker-manager|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color|langchain|langsmith|@cfworker|gpt-tokenizer|flat))[/\\\\].+\\.js$', 'packages/kbn-pm/dist/index.js', '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/[/\\\\].+\\.js$', '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/util/[/\\\\].+\\.js$', diff --git a/packages/kbn-test/jest_integration_node/jest-preset.js b/packages/kbn-test/jest_integration_node/jest-preset.js index 631b2c4f9350e..6472237c5dd17 100644 --- a/packages/kbn-test/jest_integration_node/jest-preset.js +++ b/packages/kbn-test/jest_integration_node/jest-preset.js @@ -22,7 +22,7 @@ module.exports = { // An array of regexp pattern strings that are matched against, matched files will skip transformation: transformIgnorePatterns: [ // since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842) - '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith|gpt-tokenizer))[/\\\\].+\\.js$', + '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith|gpt-tokenizer|flat))[/\\\\].+\\.js$', '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/[/\\\\].+\\.js$', '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/util/[/\\\\].+\\.js$', ], diff --git a/src/plugins/discover/public/application/main/components/top_nav/open_alerts_popover.tsx b/src/plugins/discover/public/application/main/components/top_nav/open_alerts_popover.tsx index 985b0c303cd21..ee80e467e7fb4 100644 --- a/src/plugins/discover/public/application/main/components/top_nav/open_alerts_popover.tsx +++ b/src/plugins/discover/public/application/main/components/top_nav/open_alerts_popover.tsx @@ -19,6 +19,7 @@ import { RuleCreationValidConsumer, STACK_ALERTS_FEATURE_ID, } from '@kbn/rule-data-utils'; +import { RuleTypeMetaData } from '@kbn/alerting-plugin/common'; import { DiscoverStateContainer } from '../../services/discover_state'; import { DiscoverServices } from '../../../../build_services'; @@ -42,7 +43,7 @@ interface AlertsPopoverProps { isPlainRecord?: boolean; } -interface EsQueryAlertMetaData { +interface EsQueryAlertMetaData extends RuleTypeMetaData { isManagementPage?: boolean; adHocDataViewList: DataView[]; } @@ -110,11 +111,11 @@ export function AlertsPopover({ metadata: discoverMetadata, consumer: 'alerts', onClose: (_, metadata) => { - onFinishFlyoutInteraction(metadata as EsQueryAlertMetaData); + onFinishFlyoutInteraction(metadata!); onClose(); }, onSave: async (metadata) => { - onFinishFlyoutInteraction(metadata as EsQueryAlertMetaData); + onFinishFlyoutInteraction(metadata!); }, canChangeTrigger: false, ruleTypeId: ES_QUERY_ID, diff --git a/src/plugins/unified_search/public/filter_badge/filter_badge.tsx b/src/plugins/unified_search/public/filter_badge/filter_badge.tsx index 7b20eab971e9c..e20429d5e9f36 100644 --- a/src/plugins/unified_search/public/filter_badge/filter_badge.tsx +++ b/src/plugins/unified_search/public/filter_badge/filter_badge.tsx @@ -67,7 +67,7 @@ function FilterBadge({ `} > - {!hideAlias && filter.meta.alias !== null ? ( + {filter.meta.alias && !hideAlias ? ( <> {prefix} diff --git a/test/functional/page_objects/common_page.ts b/test/functional/page_objects/common_page.ts index 653b0213bc430..98ac4d0abfe04 100644 --- a/test/functional/page_objects/common_page.ts +++ b/test/functional/page_objects/common_page.ts @@ -332,14 +332,16 @@ export class CommonPageObject extends FtrService { } currentUrl = (await this.browser.getCurrentUrl()).replace(/\/\/\w+:\w+@/, '//'); + const decodedAppUrl = decodeURIComponent(appUrl); + const decodedCurrentUrl = decodeURIComponent(currentUrl); - const navSuccessful = currentUrl + const navSuccessful = decodedCurrentUrl .replace(':80/', '/') .replace(':443/', '/') - .startsWith(appUrl.replace(':80/', '/').replace(':443/', '/')); + .startsWith(decodedAppUrl.replace(':80/', '/').replace(':443/', '/')); if (!navSuccessful) { - const msg = `App failed to load: ${appName} in ${this.defaultFindTimeout}ms appUrl=${appUrl} currentUrl=${currentUrl}`; + const msg = `App failed to load: ${appName} in ${this.defaultFindTimeout}ms appUrl=${decodedAppUrl} currentUrl=${decodedCurrentUrl}`; this.log.debug(msg); throw new Error(msg); } diff --git a/x-pack/plugins/actions/docs/openapi/bundled.json b/x-pack/plugins/actions/docs/openapi/bundled.json index d165392087670..d910d5ad6501e 100644 --- a/x-pack/plugins/actions/docs/openapi/bundled.json +++ b/x-pack/plugins/actions/docs/openapi/bundled.json @@ -2240,7 +2240,7 @@ "defaultModel": { "type": "string", "description": "The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models.\n", - "default": "anthropic.claude-v2" + "default": "anthropic.claude-v2:1" } } }, @@ -6841,4 +6841,4 @@ } } } -} \ No newline at end of file +} diff --git a/x-pack/plugins/actions/docs/openapi/bundled.yaml b/x-pack/plugins/actions/docs/openapi/bundled.yaml index 58ea32fe25764..cd55a90afa483 100644 --- a/x-pack/plugins/actions/docs/openapi/bundled.yaml +++ b/x-pack/plugins/actions/docs/openapi/bundled.yaml @@ -1498,7 +1498,7 @@ components: type: string description: | The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models. - default: anthropic.claude-v2 + default: anthropic.claude-v2:1 secrets_properties_bedrock: title: Connector secrets properties for an Amazon Bedrock connector description: Defines secrets for connectors when type is `.bedrock`. diff --git a/x-pack/plugins/actions/docs/openapi/bundled_serverless.json b/x-pack/plugins/actions/docs/openapi/bundled_serverless.json index acde35b764a5e..ba7d2b16be139 100644 --- a/x-pack/plugins/actions/docs/openapi/bundled_serverless.json +++ b/x-pack/plugins/actions/docs/openapi/bundled_serverless.json @@ -1226,7 +1226,7 @@ "defaultModel": { "type": "string", "description": "The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models.\n", - "default": "anthropic.claude-v2" + "default": "anthropic.claude-v2:1" } } }, @@ -4377,4 +4377,4 @@ } } } -} \ No newline at end of file +} diff --git a/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml b/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml index 3d9be12c8077e..564b121ec663b 100644 --- a/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml +++ b/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml @@ -857,7 +857,7 @@ components: type: string description: | The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models. - default: anthropic.claude-v2 + default: anthropic.claude-v2:1 secrets_properties_bedrock: title: Connector secrets properties for an Amazon Bedrock connector description: Defines secrets for connectors when type is `.bedrock`. diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml index 25b279c423739..189a5d5e2e05e 100644 --- a/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml @@ -12,4 +12,4 @@ properties: description: > The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models. - default: anthropic.claude-v2 \ No newline at end of file + default: anthropic.claude-v2:1 diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index 7cec5bdbdd7a6..6a66b39720402 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -19,6 +19,7 @@ export type { ActionVariable } from '@kbn/alerting-types'; export type RuleTypeState = Record; export type RuleTypeParams = Record; +export type RuleTypeMetaData = Record; // rule type defined alert fields to persist in alerts index export type RuleAlertData = Record; diff --git a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap index 70ffc475d01d6..6360d65c0e66c 100644 --- a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap +++ b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap @@ -3064,6 +3064,101 @@ Object { "presence": "optional", }, "keys": Object { + "filter": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "items": Array [ + Object { + "flags": Object { + "default": Object { + "special": "deep", + }, + "error": [Function], + "presence": "optional", + }, + "keys": Object { + "meta": Object { + "flags": Object { + "error": [Function], + }, + "rules": Array [ + Object { + "args": Object { + "key": Object { + "flags": Object { + "error": [Function], + }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "value": Object { + "flags": Object { + "error": [Function], + }, + "type": "any", + }, + }, + "name": "entries", + }, + ], + "type": "record", + }, + "query": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "rules": Array [ + Object { + "args": Object { + "key": Object { + "flags": Object { + "error": [Function], + }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "value": Object { + "flags": Object { + "error": [Function], + }, + "type": "any", + }, + }, + "name": "entries", + }, + ], + "type": "record", + }, + }, + "preferences": Object { + "stripUnknown": Object { + "objects": false, + }, + }, + "type": "object", + }, + ], + "type": "array", + }, "index": Object { "flags": Object { "error": [Function], diff --git a/x-pack/plugins/apm/public/components/alerting/ui_components/alerting_flyout/index.tsx b/x-pack/plugins/apm/public/components/alerting/ui_components/alerting_flyout/index.tsx index 95820bf8f84d4..c671bc2dda540 100644 --- a/x-pack/plugins/apm/public/components/alerting/ui_components/alerting_flyout/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/ui_components/alerting_flyout/index.tsx @@ -8,6 +8,7 @@ import React, { useCallback, useMemo } from 'react'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import { ApmRuleType } from '@kbn/rule-data-utils'; +import type { RuleTypeParams } from '@kbn/alerting-plugin/common'; import { APM_SERVER_FEATURE_ID } from '../../../../../common/rules/apm_rule_types'; import { getInitialAlertValues } from '../../utils/get_initial_alert_values'; import { ApmPluginStartDeps } from '../../../../plugin'; @@ -35,7 +36,7 @@ export function AlertingFlyout(props: Props) { const { start, end } = useTimeRange({ rangeFrom, rangeTo, optional: true }); const environment = - 'environment' in query ? query.environment : ENVIRONMENT_ALL.value; + 'environment' in query ? query.environment! : ENVIRONMENT_ALL.value; const transactionType = 'transactionType' in query ? query.transactionType : undefined; const transactionName = @@ -53,7 +54,10 @@ export function AlertingFlyout(props: Props) { const addAlertFlyout = useMemo( () => ruleType && - services.triggersActionsUi.getAddRuleFlyout({ + services.triggersActionsUi.getAddRuleFlyout< + RuleTypeParams, + AlertMetadata + >({ consumer: APM_SERVER_FEATURE_ID, onClose: onCloseAddFlyout, ruleTypeId: ruleType, @@ -67,7 +71,7 @@ export function AlertingFlyout(props: Props) { errorGroupingKey, start, end, - } as AlertMetadata, + }, useRuleProducer: true, }), /* eslint-disable-next-line react-hooks/exhaustive-deps */ diff --git a/x-pack/plugins/apm/public/components/alerting/utils/helper.ts b/x-pack/plugins/apm/public/components/alerting/utils/helper.ts index 7cc0d958aaf9e..66cfe522388f7 100644 --- a/x-pack/plugins/apm/public/components/alerting/utils/helper.ts +++ b/x-pack/plugins/apm/public/components/alerting/utils/helper.ts @@ -6,9 +6,11 @@ */ import { TIME_UNITS } from '@kbn/triggers-actions-ui-plugin/public'; +import type { RuleTypeMetaData } from '@kbn/alerting-plugin/common'; + import moment from 'moment'; -export interface AlertMetadata { +export interface AlertMetadata extends RuleTypeMetaData { environment: string; serviceName?: string; transactionType?: string; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts index 524bc70e9b279..55161f5912cfe 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts @@ -11,19 +11,43 @@ import { nextTick } from '@kbn/test-jest-helpers'; import { generateApiKey } from './generate_connector_api_key_api_logic'; +jest.mock('@kbn/search-connectors', () => ({ + createConnectorSecret: jest.fn(), + updateConnectorSecret: jest.fn(), +})); + describe('generateConnectorApiKeyApiLogic', () => { const { http } = mockHttpValues; beforeEach(() => { jest.clearAllMocks(); }); - describe('generateApiKey', () => { + describe('generateApiKey for connector clients', () => { + it('calls correct api', async () => { + const promise = Promise.resolve('result'); + http.post.mockReturnValue(promise); + const result = generateApiKey({ indexName: 'indexName', isNative: false, secretId: null }); + await nextTick(); + expect(http.post).toHaveBeenCalledWith( + '/internal/enterprise_search/indices/indexName/api_key', + { + body: '{"is_native":false,"secret_id":null}', + } + ); + await expect(result).resolves.toEqual('result'); + }); + }); + + describe('generateApiKey for native connectors', () => { it('calls correct api', async () => { const promise = Promise.resolve('result'); http.post.mockReturnValue(promise); - const result = generateApiKey({ indexName: 'indexName' }); + const result = generateApiKey({ indexName: 'indexName', isNative: true, secretId: '1234' }); await nextTick(); expect(http.post).toHaveBeenCalledWith( - '/internal/enterprise_search/indices/indexName/api_key' + '/internal/enterprise_search/indices/indexName/api_key', + { + body: '{"is_native":true,"secret_id":"1234"}', + } ); await expect(result).resolves.toEqual('result'); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts index ace963d9208be..ebca9c99add0d 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts @@ -15,9 +15,23 @@ export interface ApiKey { name: string; } -export const generateApiKey = async ({ indexName }: { indexName: string }) => { +export const generateApiKey = async ({ + indexName, + isNative, + secretId, +}: { + indexName: string; + isNative: boolean; + secretId: string | null; +}) => { const route = `/internal/enterprise_search/indices/${indexName}/api_key`; - return await HttpLogic.values.http.post(route); + const params = { + is_native: isNative, + secret_id: secretId, + }; + return await HttpLogic.values.http.post(route, { + body: JSON.stringify(params), + }); }; export const GenerateConnectorApiKeyApiLogic = createApiLogic( diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx index fdc36863f4925..b2233bb4c9ee1 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx @@ -61,10 +61,12 @@ const ConfirmModal: React.FC<{ ); -export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> = ({ - hasApiKey, - indexName, -}) => { +export const ApiKeyConfig: React.FC<{ + hasApiKey: boolean; + indexName: string; + isNative: boolean; + secretId: string | null; +}> = ({ hasApiKey, indexName, isNative, secretId }) => { const { makeRequest, apiReset } = useActions(GenerateConnectorApiKeyApiLogic); const { data, status } = useValues(GenerateConnectorApiKeyApiLogic); useEffect(() => { @@ -76,7 +78,7 @@ export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> = if (hasApiKey || data) { setIsModalVisible(true); } else { - makeRequest({ indexName }); + makeRequest({ indexName, isNative, secretId }); } }; @@ -87,7 +89,7 @@ export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> = }; const onConfirm = () => { - makeRequest({ indexName }); + makeRequest({ indexName, isNative, secretId }); setIsModalVisible(false); }; @@ -96,17 +98,28 @@ export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> = {isModalVisible && } - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.configurationConnector.apiKey.description', - { - defaultMessage: - 'First, generate an Elasticsearch API key. This {apiKeyName} key will enable read and write permissions for the connector to index documents to the created {indexName} index. Save the key in a safe place, as you will need it to configure your connector.', - values: { - apiKeyName: `${indexName}-connector`, - indexName, - }, - } - )} + {isNative + ? i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.apiKey.description', + { + defaultMessage: `This native connector's API key {apiKeyName} is managed internally by Elasticsearch. The connector uses this API key to index documents into the {indexName} index. To rollover your API key, click "Generate API key".`, + values: { + apiKeyName: `${indexName}-connector`, + indexName, + }, + } + ) + : i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.apiKey.description', + { + defaultMessage: + 'First, generate an Elasticsearch API key. This {apiKeyName} key will enable read and write permissions for the connector to index documents to the created {indexName} index. Save the key in a safe place, as you will need it to configure your connector.', + values: { + apiKeyName: `${indexName}-connector`, + indexName, + }, + } + )} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx index e338b7d1f193b..748815d4421b6 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx @@ -95,7 +95,12 @@ export const ConnectorConfiguration: React.FC = () => { steps={[ { children: ( - + ), status: hasApiKey ? 'complete' : 'incomplete', title: i18n.translate( diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx index fce32710a580d..3457e3c709fca 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx @@ -30,9 +30,11 @@ import { HttpLogic } from '../../../../../shared/http'; import { CONNECTOR_ICONS } from '../../../../../shared/icons/connector_icons'; import { KibanaLogic } from '../../../../../shared/kibana'; +import { GenerateConnectorApiKeyApiLogic } from '../../../../api/connector/generate_connector_api_key_api_logic'; import { hasConfiguredConfiguration } from '../../../../utils/has_configured_configuration'; import { isConnectorIndex } from '../../../../utils/indices'; import { IndexViewLogic } from '../../index_view_logic'; +import { ApiKeyConfig } from '../api_key_configuration'; import { ConnectorNameAndDescription } from '../connector_name_and_description/connector_name_and_description'; import { BETA_CONNECTORS, NATIVE_CONNECTORS } from '../constants'; @@ -45,6 +47,7 @@ export const NativeConnectorConfiguration: React.FC = () => { const { index } = useValues(IndexViewLogic); const { config } = useValues(KibanaLogic); const { errorConnectingMessage } = useValues(HttpLogic); + const { data: apiKeyData } = useValues(GenerateConnectorApiKeyApiLogic); if (!isConnectorIndex(index)) { return <>; @@ -74,6 +77,8 @@ export const NativeConnectorConfiguration: React.FC = () => { const hasResearched = hasDescription || hasConfigured || hasConfiguredAdvanced; const icon = nativeConnector.icon; + const hasApiKey = !!(index.connector.api_key_id ?? apiKeyData); + // TODO service_type === "" is considered unknown/custom connector multipleplaces replace all of them with a better solution const isBeta = !index.connector.service_type || @@ -140,6 +145,24 @@ export const NativeConnectorConfiguration: React.FC = () => { ), titleSize: 'xs', }, + { + children: ( + + ), + status: hasApiKey ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.regenerateApiKeyTitle', + { + defaultMessage: 'Regenerate API key', + } + ), + titleSize: 'xs', + }, { children: , status: hasDescription ? 'complete' : 'incomplete', diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts index de2d2d2db3927..61c250ebc27e5 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts @@ -11,8 +11,6 @@ import { createConnector, fetchConnectorByIndexName, deleteConnectorById, - createConnectorSecret, - updateConnectorApiKeyId, } from '@kbn/search-connectors'; import { ErrorCode } from '../../../common/types/error_codes'; @@ -27,8 +25,6 @@ jest.mock('@kbn/search-connectors', () => ({ createConnector: jest.fn(), deleteConnectorById: jest.fn(), fetchConnectorByIndexName: jest.fn(), - createConnectorSecret: jest.fn(), - updateConnectorApiKeyId: jest.fn(), })); jest.mock('../crawler/fetch_crawlers', () => ({ fetchCrawlerByIndexName: jest.fn() })); jest.mock('../indices/generate_api_key', () => ({ generateApiKey: jest.fn() })); @@ -76,10 +72,7 @@ describe('addConnector lib function', () => { (fetchConnectorByIndexName as jest.Mock).mockImplementation(() => undefined); (fetchCrawlerByIndexName as jest.Mock).mockImplementation(() => undefined); mockClient.asCurrentUser.indices.getMapping.mockImplementation(() => connectorsIndicesMapping); - (generateApiKey as jest.Mock).mockImplementation(() => undefined); - (createConnectorSecret as jest.Mock).mockImplementation(() => undefined); - (updateConnectorApiKeyId as jest.Mock).mockImplementation(() => undefined); await expect( addConnector(mockClient as unknown as IScopedClusterClient, { @@ -108,8 +101,6 @@ describe('addConnector lib function', () => { // non-native connector should not generate API key or update secrets storage expect(generateApiKey).toBeCalledTimes(0); - expect(createConnectorSecret).toBeCalledTimes(0); - expect(updateConnectorApiKeyId).toBeCalledTimes(0); }); it('should add a native connector', async () => { @@ -122,13 +113,10 @@ describe('addConnector lib function', () => { (fetchConnectorByIndexName as jest.Mock).mockImplementation(() => undefined); (fetchCrawlerByIndexName as jest.Mock).mockImplementation(() => undefined); mockClient.asCurrentUser.indices.getMapping.mockImplementation(() => connectorsIndicesMapping); - (generateApiKey as jest.Mock).mockImplementation(() => ({ id: 'api-key-id', encoded: 'encoded-api-key', })); - (createConnectorSecret as jest.Mock).mockImplementation(() => ({ id: 'connector-secret-id' })); - (updateConnectorApiKeyId as jest.Mock).mockImplementation(() => ({ acknowledged: true })); await expect( addConnector(mockClient as unknown as IScopedClusterClient, { @@ -156,14 +144,7 @@ describe('addConnector lib function', () => { }); // native connector should generate API key and update secrets storage - expect(generateApiKey).toHaveBeenCalledWith(mockClient, 'index_name'); - expect(createConnectorSecret).toHaveBeenCalledWith(mockClient.asCurrentUser, 'encoded-api-key'); - expect(updateConnectorApiKeyId).toHaveBeenCalledWith( - mockClient.asCurrentUser, - 'fakeId', - 'api-key-id', - 'connector-secret-id' - ); + expect(generateApiKey).toHaveBeenCalledWith(mockClient, 'index_name', true, null); }); it('should reject if index already exists', async () => { @@ -254,13 +235,10 @@ describe('addConnector lib function', () => { (fetchConnectorByIndexName as jest.Mock).mockImplementation(() => ({ id: 'connectorId' })); (fetchCrawlerByIndexName as jest.Mock).mockImplementation(() => undefined); mockClient.asCurrentUser.indices.getMapping.mockImplementation(() => connectorsIndicesMapping); - (generateApiKey as jest.Mock).mockImplementation(() => ({ id: 'api-key-id', encoded: 'encoded-api-key', })); - (createConnectorSecret as jest.Mock).mockImplementation(() => ({ id: 'connector-secret-id' })); - (updateConnectorApiKeyId as jest.Mock).mockImplementation(() => ({ acknowledged: true })); await expect( addConnector(mockClient as unknown as IScopedClusterClient, { diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts index 3c5265234bb9a..0f15f2767079f 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts @@ -9,11 +9,9 @@ import { IScopedClusterClient } from '@kbn/core/server'; import { createConnector, - createConnectorSecret, Connector, ConnectorStatus, deleteConnectorById, - updateConnectorApiKeyId, } from '@kbn/search-connectors'; import { fetchConnectorByIndexName, NATIVE_CONNECTOR_DEFINITIONS } from '@kbn/search-connectors'; @@ -97,14 +95,7 @@ export const addConnector = async ( input.isNative && input.serviceType !== ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE ) { - const apiKey = await generateApiKey(client, index); - const connectorSecret = await createConnectorSecret(client.asCurrentUser, apiKey.encoded); - await updateConnectorApiKeyId( - client.asCurrentUser, - connector.id, - apiKey.id, - connectorSecret.id - ); + await generateApiKey(client, index, true, null); } return connector; diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts index 92c87b354a470..541566ee2d19d 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts @@ -7,11 +7,22 @@ import { IScopedClusterClient } from '@kbn/core/server'; -import { CONNECTORS_INDEX } from '@kbn/search-connectors'; +import { + CONNECTORS_INDEX, + createConnectorSecret, + updateConnectorSecret, +} from '@kbn/search-connectors'; import { generateApiKey } from './generate_api_key'; -describe('generateApiKey lib function', () => { +jest.mock('@kbn/search-connectors', () => ({ + CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX: '.search-acl-filter-', + CONNECTORS_INDEX: '.elastic-connectors', + createConnectorSecret: jest.fn(), + updateConnectorSecret: jest.fn(), +})); + +describe('generateApiKey lib function for connector clients', () => { const mockClient = { asCurrentUser: { index: jest.fn(), @@ -47,9 +58,11 @@ describe('generateApiKey lib function', () => { encoded: 'encoded', id: 'apiKeyId', })); + (createConnectorSecret as jest.Mock).mockImplementation(() => undefined); + (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined); await expect( - generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name') + generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', false, null) ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' }); expect(mockClient.asCurrentUser.index).not.toHaveBeenCalled(); expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({ @@ -66,6 +79,8 @@ describe('generateApiKey lib function', () => { }, }, }); + expect(createConnectorSecret).toBeCalledTimes(0); + expect(updateConnectorSecret).toBeCalledTimes(0); }); it('should create an API key plus connector for connectors', async () => { mockClient.asCurrentUser.search.mockImplementation(() => @@ -83,9 +98,11 @@ describe('generateApiKey lib function', () => { encoded: 'encoded', id: 'apiKeyId', })); + (createConnectorSecret as jest.Mock).mockImplementation(() => undefined); + (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined); await expect( - generateApiKey(mockClient as unknown as IScopedClusterClient, 'search-test') + generateApiKey(mockClient as unknown as IScopedClusterClient, 'search-test', false, null) ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' }); expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({ name: 'search-test-connector', @@ -107,6 +124,8 @@ describe('generateApiKey lib function', () => { index: CONNECTORS_INDEX, }); expect(mockClient.asCurrentUser.security.invalidateApiKey).not.toHaveBeenCalled(); + expect(createConnectorSecret).toBeCalledTimes(0); + expect(updateConnectorSecret).toBeCalledTimes(0); }); it('should invalidate API key if already defined', async () => { mockClient.asCurrentUser.search.mockImplementation(() => @@ -130,9 +149,11 @@ describe('generateApiKey lib function', () => { encoded: 'encoded', id: 'apiKeyId', })); + (createConnectorSecret as jest.Mock).mockImplementation(() => undefined); + (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined); await expect( - generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name') + generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', false, null) ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' }); expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({ name: 'index_name-connector', @@ -154,7 +175,173 @@ describe('generateApiKey lib function', () => { index: CONNECTORS_INDEX, }); expect(mockClient.asCurrentUser.security.invalidateApiKey).toHaveBeenCalledWith({ - id: '1', + ids: ['1'], + }); + expect(createConnectorSecret).toBeCalledTimes(0); + expect(updateConnectorSecret).toBeCalledTimes(0); + }); +}); + +describe('generateApiKey lib function for native connectors', () => { + const mockClient = { + asCurrentUser: { + index: jest.fn(), + indices: { + create: jest.fn(), + }, + search: jest.fn(), + security: { + createApiKey: jest.fn(), + invalidateApiKey: jest.fn(), + }, + }, + asInternalUser: {}, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should create an API key if index does not have a connector', async () => { + mockClient.asCurrentUser.search.mockImplementation(() => + Promise.resolve({ + hits: { + hits: [], + }, + }) + ); + mockClient.asCurrentUser.index.mockImplementation(() => ({ + _id: 'connectorId', + _source: 'Document', + })); + mockClient.asCurrentUser.security.createApiKey.mockImplementation(() => ({ + encoded: 'encoded', + id: 'apiKeyId', + })); + (createConnectorSecret as jest.Mock).mockImplementation(() => undefined); + (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined); + + await expect( + generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', true, null) + ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' }); + expect(mockClient.asCurrentUser.index).not.toHaveBeenCalled(); + expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({ + name: 'index_name-connector', + role_descriptors: { + ['index-name-connector-role']: { + cluster: ['monitor'], + index: [ + { + names: ['index_name', '.search-acl-filter-index_name', `${CONNECTORS_INDEX}*`], + privileges: ['all'], + }, + ], + }, + }, + }); + expect(createConnectorSecret).toBeCalledTimes(0); + expect(updateConnectorSecret).toBeCalledTimes(0); + }); + it('should create an API key plus connector for connectors', async () => { + mockClient.asCurrentUser.search.mockImplementation(() => + Promise.resolve({ + hits: { + hits: [{ _id: 'connectorId', _source: { doc: 'doc' } }], + }, + }) + ); + mockClient.asCurrentUser.index.mockImplementation(() => ({ + _id: 'connectorId', + _source: 'Document', + })); + mockClient.asCurrentUser.security.createApiKey.mockImplementation(() => ({ + encoded: 'encoded', + id: 'apiKeyId', + })); + (createConnectorSecret as jest.Mock).mockImplementation(() => ({ + id: '1234', + })); + (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined); + + await expect( + generateApiKey(mockClient as unknown as IScopedClusterClient, 'search-test', true, null) + ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' }); + expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({ + name: 'search-test-connector', + role_descriptors: { + ['search-test-connector-role']: { + cluster: ['monitor'], + index: [ + { + names: ['search-test', '.search-acl-filter-search-test', `${CONNECTORS_INDEX}*`], + privileges: ['all'], + }, + ], + }, + }, + }); + expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({ + document: { api_key_id: 'apiKeyId', api_key_secret_id: '1234', doc: 'doc' }, + id: 'connectorId', + index: CONNECTORS_INDEX, + }); + expect(mockClient.asCurrentUser.security.invalidateApiKey).not.toHaveBeenCalled(); + expect(createConnectorSecret).toHaveBeenCalledWith(mockClient.asCurrentUser, 'encoded'); + expect(updateConnectorSecret).toBeCalledTimes(0); + }); + it('should invalidate API key if already defined', async () => { + mockClient.asCurrentUser.search.mockImplementation(() => + Promise.resolve({ + hits: { + hits: [ + { + _id: 'connectorId', + _source: { api_key_id: '1', doc: 'doc' }, + fields: { api_key_id: '1' }, + }, + ], + }, + }) + ); + mockClient.asCurrentUser.index.mockImplementation(() => ({ + _id: 'connectorId', + _source: 'Document', + })); + mockClient.asCurrentUser.security.createApiKey.mockImplementation(() => ({ + encoded: 'encoded', + id: 'apiKeyId', + })); + (createConnectorSecret as jest.Mock).mockImplementation(() => undefined); + (updateConnectorSecret as jest.Mock).mockImplementation(() => ({ + result: 'updated', + })); + + await expect( + generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', true, '1234') + ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' }); + expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({ + name: 'index_name-connector', + role_descriptors: { + ['index-name-connector-role']: { + cluster: ['monitor'], + index: [ + { + names: ['index_name', '.search-acl-filter-index_name', `${CONNECTORS_INDEX}*`], + privileges: ['all'], + }, + ], + }, + }, + }); + expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({ + document: { api_key_id: 'apiKeyId', api_key_secret_id: '1234', doc: 'doc' }, + id: 'connectorId', + index: CONNECTORS_INDEX, + }); + expect(mockClient.asCurrentUser.security.invalidateApiKey).toHaveBeenCalledWith({ + ids: ['1'], }); + expect(createConnectorSecret).toBeCalledTimes(0); + expect(updateConnectorSecret).toHaveBeenCalledWith(mockClient.asCurrentUser, 'encoded', '1234'); }); }); diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts index fb2ddbaad9f9d..01955cb004b24 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts @@ -11,11 +11,18 @@ import { ConnectorDocument, CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX, CONNECTORS_INDEX, + createConnectorSecret, + updateConnectorSecret, } from '@kbn/search-connectors'; import { toAlphanumeric } from '../../../common/utils/to_alphanumeric'; -export const generateApiKey = async (client: IScopedClusterClient, indexName: string) => { +export const generateApiKey = async ( + client: IScopedClusterClient, + indexName: string, + isNative: boolean, + secretId: string | null +) => { const aclIndexName = `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}${indexName}`; const apiKeyResult = await client.asCurrentUser.security.createApiKey({ @@ -32,20 +39,47 @@ export const generateApiKey = async (client: IScopedClusterClient, indexName: st }, }, }); + const connectorResult = await client.asCurrentUser.search({ index: CONNECTORS_INDEX, query: { term: { index_name: indexName } }, }); const connector = connectorResult.hits.hits[0]; if (connector) { - if (connector.fields?.api_key_id) { - await client.asCurrentUser.security.invalidateApiKey({ id: connector.fields.api_key_id }); + const apiKeyFields = isNative + ? { + api_key_id: apiKeyResult.id, + api_key_secret_id: await storeConnectorSecret(client, apiKeyResult.encoded, secretId), + } + : { + api_key_id: apiKeyResult.id, + }; + + if (connector._source?.api_key_id) { + await client.asCurrentUser.security.invalidateApiKey({ ids: [connector._source.api_key_id] }); } await client.asCurrentUser.index({ - document: { ...connector._source, api_key_id: apiKeyResult.id }, + document: { + ...connector._source, + ...apiKeyFields, + }, id: connector._id, index: CONNECTORS_INDEX, }); } return apiKeyResult; }; + +const storeConnectorSecret = async ( + client: IScopedClusterClient, + value: string, + secretId: string | null +) => { + if (secretId === null) { + const connectorSecretResult = await createConnectorSecret(client.asCurrentUser, value); + return connectorSecretResult.id; + } + + await updateConnectorSecret(client.asCurrentUser, value, secretId); + return secretId; +}; diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts index fdbeca1e82ff9..9872f4d7c7a66 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts @@ -278,6 +278,10 @@ export function registerIndexRoutes({ { path: '/internal/enterprise_search/indices/{indexName}/api_key', validate: { + body: schema.object({ + is_native: schema.boolean(), + secret_id: schema.maybe(schema.nullable(schema.string())), + }), params: schema.object({ indexName: schema.string(), }), @@ -285,9 +289,11 @@ export function registerIndexRoutes({ }, elasticsearchErrorHandler(log, async (context, request, response) => { const indexName = decodeURIComponent(request.params.indexName); + const { is_native: isNative, secret_id: secretId } = request.body; + const { client } = (await context.core).elasticsearch; - const apiKey = await generateApiKey(client, indexName); + const apiKey = await generateApiKey(client, indexName, isNative, secretId || null); return response.ok({ body: apiKey, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index f680a0bf004a6..58bcfcca386cf 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -1673,7 +1673,14 @@ describe('EPM template', () => { }, ]); - expect(esClient.indices.rollover).toHaveBeenCalled(); + expect(esClient.transport.request).toHaveBeenCalledWith( + expect.objectContaining({ + path: '/test.prefix1-default/_rollover', + querystring: { + lazy: true, + }, + }) + ); }); it('should skip rollover on expected error when flag is on', async () => { const esClient = elasticsearchServiceMock.createElasticsearchClient(); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index da2b801548e18..01b1792dc5e79 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -946,8 +946,12 @@ const getDataStreams = async ( const rolloverDataStream = (dataStreamName: string, esClient: ElasticsearchClient) => { try { // Do no wrap rollovers in retryTransientEsErrors since it is not idempotent - return esClient.indices.rollover({ - alias: dataStreamName, + return esClient.transport.request({ + method: 'POST', + path: `/${dataStreamName}/_rollover`, + querystring: { + lazy: true, + }, }); } catch (error) { throw new PackageESError( diff --git a/x-pack/plugins/index_management/common/lib/template_serialization.ts b/x-pack/plugins/index_management/common/lib/template_serialization.ts index 5ec150a85aa17..80a5514969f54 100644 --- a/x-pack/plugins/index_management/common/lib/template_serialization.ts +++ b/x-pack/plugins/index_management/common/lib/template_serialization.ts @@ -23,6 +23,7 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T indexPatterns, template, composedOf, + ignoreMissingComponentTemplates, dataStream, _meta, allowAutoCreate, @@ -35,6 +36,7 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T index_patterns: indexPatterns, data_stream: dataStream, composed_of: composedOf, + ignore_missing_component_templates: ignoreMissingComponentTemplates, allow_auto_create: allowAutoCreate, _meta, }; @@ -52,6 +54,7 @@ export function deserializeTemplate( priority, _meta, composed_of: composedOf, + ignore_missing_component_templates: ignoreMissingComponentTemplates, data_stream: dataStream, deprecated, allow_auto_create: allowAutoCreate, @@ -76,6 +79,7 @@ export function deserializeTemplate( template, ilmPolicy: settings?.index?.lifecycle, composedOf: composedOf ?? [], + ignoreMissingComponentTemplates: ignoreMissingComponentTemplates ?? [], dataStream, allowAutoCreate, _meta, diff --git a/x-pack/plugins/index_management/common/types/templates.ts b/x-pack/plugins/index_management/common/types/templates.ts index e2f530b4ad502..9205605c70010 100644 --- a/x-pack/plugins/index_management/common/types/templates.ts +++ b/x-pack/plugins/index_management/common/types/templates.ts @@ -23,6 +23,7 @@ export interface TemplateSerialized { }; deprecated?: boolean; composed_of?: string[]; + ignore_missing_component_templates?: string[]; version?: number; priority?: number; _meta?: { [key: string]: any }; @@ -45,6 +46,7 @@ export interface TemplateDeserialized { }; lifecycle?: DataRetention; composedOf?: string[]; // Composable template only + ignoreMissingComponentTemplates?: string[]; version?: number; priority?: number; // Composable template only allowAutoCreate?: boolean; diff --git a/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts b/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts index b9020585ed676..1f5c5e2a3b82e 100644 --- a/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts +++ b/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts @@ -28,6 +28,7 @@ export const templateSchema = schema.object({ }) ), composedOf: schema.maybe(schema.arrayOf(schema.string())), + ignoreMissingComponentTemplates: schema.maybe(schema.arrayOf(schema.string())), dataStream: schema.maybe( schema.object( { diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx index 1fc8cc6614a75..4b7a1907e206a 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx @@ -4,9 +4,9 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; -import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, type EuiAccordionProps } from '@elastic/eui'; import { useSummaryTimeRange } from '@kbn/observability-plugin/public'; import type { TimeRange } from '@kbn/es-query'; import type { InventoryItemType } from '@kbn/metrics-data-access-plugin/common'; @@ -24,6 +24,8 @@ import { ALERT_STATUS_ALL } from '../../../../common/alerts/constants'; import { AlertsSectionTitle } from '../../components/section_titles'; import { useAssetDetailsRenderPropsContext } from '../../hooks/use_asset_details_render_props'; import { CollapsibleSection } from './section/collapsible_section'; +import { AlertsClosedContent } from './alerts_closed_content'; +import { type AlertsCount } from '../../../../hooks/use_alerts_count'; export const AlertsSummaryContent = ({ assetName, @@ -37,6 +39,9 @@ export const AlertsSummaryContent = ({ const { featureFlags } = usePluginConfig(); const [isAlertFlyoutVisible, { toggle: toggleAlertFlyout }] = useBoolean(false); const { overrides } = useAssetDetailsRenderPropsContext(); + const [collapsibleStatus, setCollapsibleStatus] = + useState('open'); + const [activeAlertsCount, setActiveAlertsCount] = useState(undefined); const alertsEsQueryByStatus = useMemo( () => @@ -48,6 +53,14 @@ export const AlertsSummaryContent = ({ [assetName, dateRange] ); + const onLoaded = (alertsCount?: AlertsCount) => { + const { activeAlertCount = 0 } = alertsCount ?? {}; + const hasActiveAlerts = activeAlertCount > 0; + + setCollapsibleStatus(hasActiveAlerts ? 'open' : 'closed'); + setActiveAlertsCount(alertsCount?.activeAlertCount); + }; + return ( <> } + initialTriggerValue={collapsibleStatus} extraAction={ {featureFlags.inventoryThresholdAlertRuleEnabled && ( @@ -72,9 +87,12 @@ export const AlertsSummaryContent = ({ } > - + - {featureFlags.inventoryThresholdAlertRuleEnabled && ( void; } const MemoAlertSummaryWidget = React.memo( - ({ alertsQuery, dateRange }: MemoAlertSummaryWidgetProps) => { + ({ alertsQuery, dateRange, onLoaded }: MemoAlertSummaryWidgetProps) => { const { services } = useKibanaContextForPlugin(); const summaryTimeRange = useSummaryTimeRange(dateRange); @@ -112,6 +131,7 @@ const MemoAlertSummaryWidget = React.memo( featureIds={infraAlertFeatureIds} filter={alertsQuery} timeRange={summaryTimeRange} + onLoaded={onLoaded} fullSize hideChart /> diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts_closed_content.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts_closed_content.tsx new file mode 100644 index 0000000000000..a08a0313230ee --- /dev/null +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts_closed_content.tsx @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiBadge, EuiToolTip } from '@elastic/eui'; + +export const AlertsClosedContent = ({ activeAlertCount }: { activeAlertCount?: number }) => { + const shouldRenderAlertsClosedContent = typeof activeAlertCount === 'number'; + + if (!shouldRenderAlertsClosedContent) { + return null; + } + + if (activeAlertCount > 0) { + return ( + + + {activeAlertCount} + + + ); + } + + return ( + + {i18n.translate('xpack.infra.assetDetails.noActiveAlertsContentClosedSection', { + defaultMessage: 'No active alerts', + })} + + ); +}; diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx index ec31851d89a6d..da0b993199ee1 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx @@ -5,6 +5,8 @@ * 2.0. */ +import React, { useEffect, useState } from 'react'; + import { EuiAccordion, EuiFlexGroup, @@ -12,7 +14,6 @@ import { useGeneratedHtmlId, type EuiAccordionProps, } from '@elastic/eui'; -import React, { useState } from 'react'; export const CollapsibleSection = ({ title, @@ -22,6 +23,7 @@ export const CollapsibleSection = ({ collapsible, ['data-test-subj']: dataTestSubj, id, + initialTriggerValue, }: { title: React.FunctionComponent; closedSectionContent?: React.ReactNode; @@ -31,13 +33,18 @@ export const CollapsibleSection = ({ collapsible: boolean; ['data-test-subj']: string; id: string; + initialTriggerValue?: EuiAccordionProps['forceState']; }) => { const [trigger, setTrigger] = useState('open'); + useEffect(() => { + setTrigger(initialTriggerValue ?? 'open'); + }, [initialTriggerValue]); + const Title = title; const ButtonContent = () => closedSectionContent && trigger === 'closed' ? ( - + </EuiFlexItem> diff --git a/x-pack/plugins/infra/public/hooks/use_alerts_count.ts b/x-pack/plugins/infra/public/hooks/use_alerts_count.ts index 7d05a275d6eae..5c602d09b7d23 100644 --- a/x-pack/plugins/infra/public/hooks/use_alerts_count.ts +++ b/x-pack/plugins/infra/public/hooks/use_alerts_count.ts @@ -28,7 +28,7 @@ interface FetchAlertsCountParams { signal: AbortSignal; } -interface AlertsCount { +export interface AlertsCount { activeAlertCount: number; recoveredAlertCount: number; } diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx index 15c4e568ad8ed..c3f778299ab69 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx @@ -5,7 +5,7 @@ * 2.0. */ import React from 'react'; -import { EuiIcon, EuiLoadingSpinner, EuiNotificationBadge, EuiToolTip } from '@elastic/eui'; +import { EuiIcon, EuiLoadingSpinner, EuiBadge, EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { useAlertsCount } from '../../../../../hooks/use_alerts_count'; import { infraAlertFeatureIds } from './config'; @@ -40,12 +40,12 @@ export const AlertsTabBadge = () => { typeof alertsCount?.activeAlertCount === 'number' && alertsCount.activeAlertCount > 0; return shouldRenderBadge ? ( - <EuiNotificationBadge + <EuiBadge + color="danger" className="eui-alignCenter" - size="m" data-test-subj="hostsView-tabs-alerts-count" > {alertsCount?.activeAlertCount} - </EuiNotificationBadge> + </EuiBadge> ) : null; }; diff --git a/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx b/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx index d436a52f7ccb3..22500f936e705 100644 --- a/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx +++ b/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx @@ -21,6 +21,7 @@ import { EuiText, EuiSpacer, } from '@elastic/eui'; +import useDebounce from 'react-use/lib/useDebounce'; import type { CanDeleteMLSpaceAwareItemsResponse, MlSavedObjectType, @@ -246,12 +247,16 @@ export const DeleteSpaceAwareItemCheckModal: FC<Props> = ({ const [itemCheckRespSummary, setItemCheckRespSummary] = useState< CanDeleteMLSpaceAwareItemsSummary | undefined >(); + const [showModal, setShowModal] = useState<boolean>(false); const { savedObjects: { canDeleteMLSpaceAwareItems, removeItemFromCurrentSpace }, } = useMlApiContext(); const { displayErrorToast, displaySuccessToast } = useToastNotificationService(); + // delay showing the modal to avoid flickering + useDebounce(() => setShowModal(true), 1000); + useEffect(() => { setIsLoading(true); // Do the spaces check and set the content for the modal and buttons depending on results @@ -321,6 +326,10 @@ export const DeleteSpaceAwareItemCheckModal: FC<Props> = ({ } }; + if (showModal === false) { + return null; + } + return ( <EuiModal onClose={onCloseCallback} data-test-subj="mlDeleteSpaceAwareItemCheckModalOverlay"> <> diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/__snapshots__/alert_details_app_section.test.tsx.snap b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/__snapshots__/alert_details_app_section.test.tsx.snap index b963137281b70..a9a77b477f1a2 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/__snapshots__/alert_details_app_section.test.tsx.snap +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/__snapshots__/alert_details_app_section.test.tsx.snap @@ -28,7 +28,6 @@ Array [ }, ], "dataView": undefined, - "filterQuery": "", "groupBy": Array [ "host.hostname", ], @@ -46,6 +45,13 @@ Array [ "timeSize": 15, "timeUnit": "m", }, + "searchConfiguration": Object { + "index": "mockedIndex", + "query": Object { + "language": "kuery", + "query": "host.hostname: Users-System.local and service.type: system", + }, + }, "seriesType": "bar_stacked", "timeRange": Object { "from": "2023-03-28T10:43:13.802Z", diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.tsx b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.tsx index 2506516efd81a..f07a6ddac4501 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.tsx +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.tsx @@ -5,7 +5,6 @@ * 2.0. */ -import { Query } from '@kbn/es-query'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import React, { useEffect, useState } from 'react'; @@ -54,7 +53,6 @@ import { LogRateAnalysis } from './log_rate_analysis'; import { Groups } from './groups'; import { Tags } from './tags'; import { RuleConditionChart } from '../rule_condition_chart/rule_condition_chart'; -import { getFilterQuery } from './helpers/get_filter_query'; // TODO Use a generic props for app sections https://github.com/elastic/kibana/issues/152690 export type CustomThresholdRule = Rule<CustomThresholdRuleTypeParams>; @@ -118,7 +116,6 @@ export default function AlertDetailsAppSection({ const { euiTheme } = useEuiTheme(); const hasLogRateAnalysisLicense = hasAtLeast('platinum'); const [dataView, setDataView] = useState<DataView>(); - const [filterQuery, setFilterQuery] = useState<string>(''); const [, setDataViewError] = useState<Error>(); const ruleParams = rule.params as RuleTypeParams & AlertParams; const chartProps = { @@ -204,11 +201,6 @@ export default function AlertDetailsAppSection({ setAlertSummaryFields(alertSummaryFields); }, [groups, tags, rule, ruleLink, setAlertSummaryFields]); - useEffect(() => { - const query = `${(ruleParams.searchConfiguration?.query as Query)?.query as string}`; - setFilterQuery(getFilterQuery(query, groups)); - }, [groups, ruleParams.searchConfiguration]); - useEffect(() => { const initDataView = async () => { const ruleSearchConfiguration = ruleParams.searchConfiguration; @@ -271,7 +263,7 @@ export default function AlertDetailsAppSection({ <RuleConditionChart metricExpression={criterion} dataView={dataView} - filterQuery={filterQuery} + searchConfiguration={ruleParams.searchConfiguration} groupBy={ruleParams.groupBy} annotations={annotations} timeRange={timeRange} diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.test.tsx b/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.test.tsx index 7aab6dd0d636b..d07eb08de4474 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.test.tsx +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.test.tsx @@ -6,6 +6,7 @@ */ import React from 'react'; +import { SerializedSearchSourceFields } from '@kbn/data-plugin/common'; import { act } from 'react-dom/test-utils'; import { DataView } from '@kbn/data-views-plugin/common'; import { mountWithIntl, nextTick } from '@kbn/test-jest-helpers'; @@ -35,7 +36,7 @@ describe('Rule condition chart', () => { <RuleConditionChart metricExpression={expression} dataView={dataView} - filterQuery={''} + searchConfiguration={{} as SerializedSearchSourceFields} groupBy={[]} error={{}} timeRange={{ from: 'now-15m', to: 'now' }} diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.tsx b/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.tsx index e1eefcfc2f706..3cf10659ee66a 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.tsx +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/rule_condition_chart/rule_condition_chart.tsx @@ -4,8 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import React, { useState, useEffect } from 'react'; +import { SerializedSearchSourceFields } from '@kbn/data-plugin/common'; import { EuiEmptyPrompt, useEuiTheme } from '@elastic/eui'; +import { Query } from '@kbn/es-query'; import { FillStyle, SeriesType } from '@kbn/lens-plugin/public'; import { DataView } from '@kbn/data-views-plugin/common'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -38,8 +41,8 @@ import { interface RuleConditionChartProps { metricExpression: MetricExpression; + searchConfiguration: SerializedSearchSourceFields; dataView?: DataView; - filterQuery?: string; groupBy?: string | string[]; error?: IErrorObject; timeRange: TimeRange; @@ -47,10 +50,15 @@ interface RuleConditionChartProps { seriesType?: SeriesType; } +const defaultQuery: Query = { + language: 'kuery', + query: '', +}; + export function RuleConditionChart({ metricExpression, + searchConfiguration, dataView, - filterQuery, groupBy, error, annotations, @@ -283,7 +291,7 @@ export function RuleConditionChart({ comparator, dataView, equation, - filterQuery, + searchConfiguration, formula, formulaAsync.value, groupBy, @@ -337,10 +345,8 @@ export function RuleConditionChart({ timeRange={timeRange} attributes={attributes} disableTriggers={true} - query={{ - language: 'kuery', - query: filterQuery || '', - }} + query={(searchConfiguration.query as Query) || defaultQuery} + filters={searchConfiguration.filter} /> </div> ); diff --git a/x-pack/plugins/observability/public/components/custom_threshold/custom_threshold_rule_expression.tsx b/x-pack/plugins/observability/public/components/custom_threshold/custom_threshold_rule_expression.tsx index cd93a5e134c2b..19a1c0fcd164e 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/custom_threshold_rule_expression.tsx +++ b/x-pack/plugins/observability/public/components/custom_threshold/custom_threshold_rule_expression.tsx @@ -405,7 +405,7 @@ export default function Expressions(props: Props) { indexPatterns={dataView ? [dataView] : undefined} showQueryInput={true} showQueryMenu={false} - showFilterBar={false} + showFilterBar={!!ruleParams.searchConfiguration?.filter} showDatePicker={false} showSubmitButton={false} displayStyle="inPage" @@ -413,6 +413,16 @@ export default function Expressions(props: Props) { onQuerySubmit={onFilterChange} dataTestSubj="thresholdRuleUnifiedSearchBar" query={ruleParams.searchConfiguration?.query as Query} + filters={ruleParams.searchConfiguration?.filter} + onFiltersUpdated={(filter) => { + // Since rule params will be sent to the API as is, and we only need meta and query parameters to be + // saved in the rule's saved object, we filter extra fields here (such as $state). + const filters = filter.map(({ meta, query }) => ({ meta, query })); + setRuleParams('searchConfiguration', { + ...ruleParams.searchConfiguration, + filter: filters, + }); + }} /> {errors.filterQuery && ( <EuiFormErrorText data-test-subj="thresholdRuleDataViewErrorNoTimestamp"> @@ -454,7 +464,7 @@ export default function Expressions(props: Props) { <PreviewChart metricExpression={e} dataView={dataView} - filterQuery={(ruleParams.searchConfiguration?.query as Query)?.query as string} + searchConfiguration={ruleParams.searchConfiguration} groupBy={ruleParams.groupBy} error={(errors[idx] as IErrorObject) || emptyError} timeRange={{ from: `now-${(timeSize ?? 1) * 20}${timeUnit}`, to: 'now' }} diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/check_missing_group.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/check_missing_group.ts index c45120cc62fa3..45bb92426eca8 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/check_missing_group.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/check_missing_group.ts @@ -6,11 +6,13 @@ */ import { ElasticsearchClient } from '@kbn/core/server'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { Logger } from '@kbn/logging'; import { isString, get, identity } from 'lodash'; +import { SearchConfigurationType } from '../types'; import { CustomMetricExpressionParams } from '../../../../../common/custom_threshold_rule/types'; import type { BucketKey } from './get_data'; -import { calculateCurrentTimeFrame, createBaseFilters } from './metric_query'; +import { calculateCurrentTimeFrame, createBoolQuery } from './metric_query'; export interface MissingGroupsRecord { key: string; @@ -23,7 +25,7 @@ export const checkMissingGroups = async ( indexPattern: string, timeFieldName: string, groupBy: string | undefined | string[], - filterQuery: string | undefined, + searchConfiguration: SearchConfigurationType, logger: Logger, timeframe: { start: number; end: number }, missingGroups: MissingGroupsRecord[] = [] @@ -32,28 +34,31 @@ export const checkMissingGroups = async ( return missingGroups; } const currentTimeFrame = calculateCurrentTimeFrame(metricParams, timeframe); - const baseFilters = createBaseFilters(currentTimeFrame, timeFieldName, filterQuery); const groupByFields = isString(groupBy) ? [groupBy] : groupBy ? groupBy : []; const searches = missingGroups.flatMap((group) => { - const groupByFilters = Object.values(group.bucketKey).map((key, index) => { - return { - match: { - [groupByFields[index]]: key, - }, - }; - }); + const groupByQueries: QueryDslQueryContainer[] = Object.values(group.bucketKey).map( + (key, index) => { + return { + match: { + [groupByFields[index]]: key, + }, + }; + } + ); + const query = createBoolQuery( + currentTimeFrame, + timeFieldName, + searchConfiguration, + groupByQueries + ); return [ { index: indexPattern }, { size: 0, terminate_after: 1, track_total_hits: true, - query: { - bool: { - filter: [...baseFilters, ...groupByFilters], - }, - }, + query, }, ]; }); diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts index 59f5801613dd0..87b7d9983465b 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts @@ -69,7 +69,7 @@ export const evaluateRule = async <Params extends EvaluatedRuleParams = Evaluate dataView, timeFieldName, groupBy, - searchConfiguration.query.query, + searchConfiguration, compositeSize, alertOnGroupDisappear, calculatedTimerange, @@ -83,7 +83,7 @@ export const evaluateRule = async <Params extends EvaluatedRuleParams = Evaluate dataView, timeFieldName, groupBy, - searchConfiguration.query.query, + searchConfiguration, logger, calculatedTimerange, missingGroups diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/get_data.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/get_data.ts index 9658e0876ca1e..1e488ffc57060 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/get_data.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/get_data.ts @@ -9,6 +9,7 @@ import { SearchResponse, AggregationsAggregate } from '@elastic/elasticsearch/li import { ElasticsearchClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common/search_strategy'; +import { SearchConfigurationType } from '../types'; import { CustomMetricExpressionParams } from '../../../../../common/custom_threshold_rule/types'; import { UNGROUPED_FACTORY_KEY } from '../constants'; @@ -100,7 +101,7 @@ export const getData = async ( index: string, timeFieldName: string, groupBy: string | undefined | string[], - filterQuery: string | undefined, + searchConfiguration: SearchConfigurationType, compositeSize: number, alertOnGroupDisappear: boolean, timeframe: { start: number; end: number }, @@ -159,7 +160,7 @@ export const getData = async ( index, timeFieldName, groupBy, - filterQuery, + searchConfiguration, compositeSize, alertOnGroupDisappear, timeframe, @@ -202,9 +203,9 @@ export const getData = async ( timeFieldName, compositeSize, alertOnGroupDisappear, + searchConfiguration, lastPeriodEnd, groupBy, - filterQuery, afterKey, fieldsExisted ), diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.test.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.test.ts index 01d94dd5f7d67..c12f14a7aeb5b 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.test.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.test.ts @@ -11,6 +11,7 @@ import { Aggregators, CustomMetricExpressionParams, } from '../../../../../common/custom_threshold_rule/types'; +import { SearchConfigurationType } from '../types'; import { getElasticsearchMetricQuery } from './metric_query'; describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { @@ -27,6 +28,18 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { threshold: [1], comparator: Comparator.GT, }; + const searchConfiguration: SearchConfigurationType = { + index: { + id: 'dataset-logs-*-*', + name: 'All logs', + timeFieldName: '@timestamp', + title: 'logs-*-*', + }, + query: { + language: 'kuery', + query: '', + }, + }; const groupBy = 'host.doggoname'; const timeFieldName = 'mockedTimeFieldName'; @@ -35,13 +48,14 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { end: moment().valueOf(), }; - describe('when passed no filterQuery', () => { + describe('when passed no KQL query', () => { const searchBody = getElasticsearchMetricQuery( expressionParams, timeframe, timeFieldName, 100, true, + searchConfiguration, void 0, groupBy ); @@ -78,11 +92,18 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { }); }); - describe('when passed a filterQuery', () => { + describe('when passed a KQL query', () => { // This is adapted from a real-world query that previously broke alerts // We want to make sure it doesn't override any existing filters // https://github.com/elastic/kibana/issues/68492 - const filterQuery = 'NOT host.name:dv* and NOT host.name:ts*'; + const query = 'NOT host.name:dv* and NOT host.name:ts*'; + const currentSearchConfiguration = { + ...searchConfiguration, + query: { + language: 'kuery', + query, + }, + }; const searchBody = getElasticsearchMetricQuery( expressionParams, @@ -90,9 +111,9 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { timeFieldName, 100, true, + currentSearchConfiguration, void 0, - groupBy, - filterQuery + groupBy ); test('includes a range filter', () => { expect( @@ -164,4 +185,60 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { ); }); }); + + describe('when passed a filter', () => { + const currentSearchConfiguration = { + ...searchConfiguration, + query: { + language: 'kuery', + query: '', + }, + filter: [ + { + meta: { + alias: null, + disabled: false, + field: 'service.name', + key: 'service.name', + negate: false, + params: { + query: 'synth-node-2', + }, + type: 'phrase', + index: 'dataset-logs-*-*', + }, + query: { + match_phrase: { + 'service.name': 'synth-node-2', + }, + }, + }, + ], + }; + + const searchBody = getElasticsearchMetricQuery( + expressionParams, + timeframe, + timeFieldName, + 100, + true, + currentSearchConfiguration, + void 0, + groupBy + ); + test('includes a range filter', () => { + expect( + searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range')) + ).toBeTruthy(); + }); + + test('includes a metric field filter', () => { + expect(searchBody.query.bool.filter).toMatchObject( + expect.arrayContaining([ + { range: { mockedTimeFieldName: expect.any(Object) } }, + { match_phrase: { 'service.name': 'synth-node-2' } }, + ]) + ); + }); + }); }); diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.ts index 3cc1eee92fec9..14c18e4af1334 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/metric_query.ts @@ -6,10 +6,14 @@ */ import moment from 'moment'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { Filter } from '@kbn/es-query'; import { Aggregators, CustomMetricExpressionParams, } from '../../../../../common/custom_threshold_rule/types'; +import { getSearchConfigurationBoolQuery } from '../../../../utils/get_parsed_filtered_query'; +import { SearchConfigurationType } from '../types'; import { createCustomMetricsAggregations } from './create_custom_metrics_aggregations'; import { CONTAINER_ID, @@ -20,7 +24,6 @@ import { } from '../utils'; import { createBucketSelector } from './create_bucket_selector'; import { wrapInCurrentPeriod } from './wrap_in_period'; -import { getParsedFilterQuery } from '../../../../utils/get_parsed_filtered_query'; export const calculateCurrentTimeFrame = ( metricParams: CustomMetricExpressionParams, @@ -38,25 +41,30 @@ export const calculateCurrentTimeFrame = ( }; }; -export const createBaseFilters = ( +const QueryDslQueryContainerToFilter = (queries: QueryDslQueryContainer[]): Filter[] => { + return queries.map((query) => ({ + meta: {}, + query, + })); +}; + +export const createBoolQuery = ( timeframe: { start: number; end: number }, timeFieldName: string, - filterQuery?: string + searchConfiguration: SearchConfigurationType, + additionalQueries: QueryDslQueryContainer[] = [] ) => { - const rangeFilters = [ - { - range: { - [timeFieldName]: { - gte: moment(timeframe.start).toISOString(), - lte: moment(timeframe.end).toISOString(), - }, + const rangeQuery: QueryDslQueryContainer = { + range: { + [timeFieldName]: { + gte: moment(timeframe.start).toISOString(), + lte: moment(timeframe.end).toISOString(), }, }, - ]; - - const parsedFilterQuery = getParsedFilterQuery(filterQuery); + }; + const filters = QueryDslQueryContainerToFilter([rangeQuery, ...additionalQueries]); - return [...rangeFilters, ...parsedFilterQuery]; + return getSearchConfigurationBoolQuery(searchConfiguration, filters); }; export const getElasticsearchMetricQuery = ( @@ -65,9 +73,9 @@ export const getElasticsearchMetricQuery = ( timeFieldName: string, compositeSize: number, alertOnGroupDisappear: boolean, + searchConfiguration: SearchConfigurationType, lastPeriodEnd?: number, groupBy?: string | string[], - filterQuery?: string, afterKey?: Record<string, string>, fieldsExisted?: Record<string, boolean> | null ) => { @@ -196,15 +204,11 @@ export const getElasticsearchMetricQuery = ( aggs.groupings.composite.after = afterKey; } - const baseFilters = createBaseFilters(timeframe, timeFieldName, filterQuery); + const query = createBoolQuery(timeframe, timeFieldName, searchConfiguration); return { track_total_hits: true, - query: { - bool: { - filter: baseFilters, - }, - }, + query, size: 0, aggs, }; diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/register_custom_threshold_rule_type.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/register_custom_threshold_rule_type.ts index 5e9c2e0cea019..df64a67ca8e4a 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/register_custom_threshold_rule_type.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/register_custom_threshold_rule_type.ts @@ -58,6 +58,14 @@ export const searchConfigurationSchema = schema.object({ }), query: schema.string(), }), + filter: schema.maybe( + schema.arrayOf( + schema.object({ + query: schema.maybe(schema.recordOf(schema.string(), schema.any())), + meta: schema.recordOf(schema.string(), schema.any()), + }) + ) + ), }); type CreateLifecycleExecutor = ReturnType<typeof createLifecycleExecutor>; diff --git a/x-pack/plugins/observability/server/utils/get_parsed_filtered_query.ts b/x-pack/plugins/observability/server/utils/get_parsed_filtered_query.ts index fabefa63f0695..033a6cadc282e 100644 --- a/x-pack/plugins/observability/server/utils/get_parsed_filtered_query.ts +++ b/x-pack/plugins/observability/server/utils/get_parsed_filtered_query.ts @@ -5,7 +5,14 @@ * 2.0. */ -import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; +import { + BoolQuery, + buildEsQuery, + Filter, + fromKueryExpression, + toElasticsearchQuery, +} from '@kbn/es-query'; +import { SearchConfigurationType } from '../lib/rules/custom_threshold/types'; export const getParsedFilterQuery: (filter: string | undefined) => Array<Record<string, any>> = ( filter @@ -19,3 +26,24 @@ export const getParsedFilterQuery: (filter: string | undefined) => Array<Record< return []; } }; + +export const getSearchConfigurationBoolQuery: ( + searchConfiguration: SearchConfigurationType, + additionalFilters: Filter[] +) => { bool: BoolQuery } = (searchConfiguration, additionalFilters) => { + try { + const searchConfigurationFilters = (searchConfiguration.filter as Filter[]) || []; + const filters = [...additionalFilters, ...searchConfigurationFilters]; + + return buildEsQuery(undefined, searchConfiguration.query, filters, {}); + } catch (error) { + return { + bool: { + must: [], + must_not: [], + filter: [], + should: [], + }, + }; + } +}; diff --git a/x-pack/plugins/observability_ai_assistant/common/connectors.ts b/x-pack/plugins/observability_ai_assistant/common/connectors.ts new file mode 100644 index 0000000000000..2b834081a7ac9 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/common/connectors.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum ObservabilityAIAssistantConnectorType { + Bedrock = '.bedrock', + OpenAI = '.gen-ai', +} + +export const SUPPORTED_CONNECTOR_TYPES = [ + ObservabilityAIAssistantConnectorType.OpenAI, + ObservabilityAIAssistantConnectorType.Bedrock, +]; + +export function isSupportedConnectorType( + type: string +): type is ObservabilityAIAssistantConnectorType { + return ( + type === ObservabilityAIAssistantConnectorType.Bedrock || + type === ObservabilityAIAssistantConnectorType.OpenAI + ); +} diff --git a/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts b/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts index f5fe0d37408c2..b082478bba100 100644 --- a/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts +++ b/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts @@ -14,6 +14,7 @@ export enum StreamingChatResponseEventType { ConversationUpdate = 'conversationUpdate', MessageAdd = 'messageAdd', ChatCompletionError = 'chatCompletionError', + BufferFlush = 'bufferFlush', } type StreamingChatResponseEventBase< @@ -76,6 +77,13 @@ export type ChatCompletionErrorEvent = StreamingChatResponseEventBase< } >; +export type BufferFlushEvent = StreamingChatResponseEventBase< + StreamingChatResponseEventType.BufferFlush, + { + data?: string; + } +>; + export type StreamingChatResponseEvent = | ChatCompletionChunkEvent | ConversationCreateEvent @@ -129,7 +137,14 @@ export function createConversationNotFoundError() { ); } -export function createInternalServerError(originalErrorMessage: string) { +export function createInternalServerError( + originalErrorMessage: string = i18n.translate( + 'xpack.observabilityAiAssistant.chatCompletionError.internalServerError', + { + defaultMessage: 'An internal server error occurred', + } + ) +) { return new ChatCompletionError(ChatCompletionErrorCode.InternalError, originalErrorMessage); } diff --git a/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts b/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts index 2487fca287cc7..8b6ef27ee8ebd 100644 --- a/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts +++ b/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts @@ -19,7 +19,6 @@ export function processOpenAiStream() { const id = v4(); return source.pipe( - map((line) => line.substring(6)), filter((line) => !!line && line !== '[DONE]'), map( (line) => diff --git a/x-pack/plugins/observability_ai_assistant/kibana.jsonc b/x-pack/plugins/observability_ai_assistant/kibana.jsonc index 3f346cccff0c1..a3eaad0d216a3 100644 --- a/x-pack/plugins/observability_ai_assistant/kibana.jsonc +++ b/x-pack/plugins/observability_ai_assistant/kibana.jsonc @@ -25,7 +25,9 @@ "ml" ], "requiredBundles": [ "kibanaReact", "kibanaUtils"], - "optionalPlugins": [], + "optionalPlugins": [ + "cloud" + ], "extraPublicDirs": [] } } diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx index 8ed26d71acc58..a89419c366a2d 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx @@ -35,11 +35,7 @@ import { ChatTimeline } from './chat_timeline'; import { Feedback } from '../feedback_buttons'; import { IncorrectLicensePanel } from './incorrect_license_panel'; import { WelcomeMessage } from './welcome_message'; -import { - ChatActionClickHandler, - ChatActionClickType, - type ChatFlyoutSecondSlotHandler, -} from './types'; +import { ChatActionClickHandler, ChatActionClickType } from './types'; import { ASSISTANT_SETUP_TITLE, EMPTY_CONVERSATION_TITLE, UPGRADE_LICENSE_TITLE } from '../../i18n'; import type { StartedFrom } from '../../utils/get_timeline_items_from_conversation'; import { TELEMETRY, sendEvent } from '../../analytics'; @@ -94,7 +90,6 @@ const animClassName = css` const PADDING_AND_BORDER = 32; export function ChatBody({ - chatFlyoutSecondSlotHandler, connectors, currentUser, flyoutWidthMode, @@ -107,7 +102,6 @@ export function ChatBody({ onConversationUpdate, onToggleFlyoutWidthMode, }: { - chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler; connectors: UseGenAIConnectorsResult; currentUser?: Pick<AuthenticatedUser, 'full_name' | 'username'>; flyoutWidthMode?: FlyoutWidthMode; @@ -362,7 +356,6 @@ export function ChatBody({ onStopGenerating={() => { stop(); }} - chatFlyoutSecondSlotHandler={chatFlyoutSecondSlotHandler} onActionClick={handleActionClick} /> )} diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx index 6823153397ca4..5a8b0ee3b3776 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx @@ -18,6 +18,7 @@ import { EuiToolTip, useEuiTheme, } from '@elastic/eui'; +import { ObservabilityAIAssistantMultipaneFlyoutProvider } from '../../context/observability_ai_assistant_multipane_flyout_provider'; import { useForceUpdate } from '../../hooks/use_force_update'; import { useCurrentUser } from '../../hooks/use_current_user'; import { useGenAIConnectors } from '../../hooks/use_genai_connectors'; @@ -134,133 +135,136 @@ export function ChatFlyout({ }; return isOpen ? ( - <EuiFlyout - closeButtonProps={{ - css: { marginRight: `${euiTheme.size.s}`, marginTop: `${euiTheme.size.s}` }, - }} - size={getFlyoutWidth({ - expanded: conversationsExpanded, - isSecondSlotVisible, - flyoutWidthMode, - })} - paddingSize="m" - onClose={() => { - onClose(); - setIsSecondSlotVisible(false); - if (secondSlotContainer) { - ReactDOM.unmountComponentAtNode(secondSlotContainer); - } + <ObservabilityAIAssistantMultipaneFlyoutProvider + value={{ + container: secondSlotContainer, + setVisibility: setIsSecondSlotVisible, }} > - <EuiFlexGroup gutterSize="none" className={containerClassName}> - <EuiFlexItem className={sidebarClass}> - <EuiPopover - anchorPosition="downLeft" - className={expandButtonContainerClassName} - button={ - <EuiToolTip - content={ - conversationsExpanded - ? i18n.translate( - 'xpack.observabilityAiAssistant.chatFlyout.euiToolTip.collapseConversationListLabel', - { defaultMessage: 'Collapse conversation list' } - ) - : i18n.translate( - 'xpack.observabilityAiAssistant.chatFlyout.euiToolTip.expandConversationListLabel', - { defaultMessage: 'Expand conversation list' } - ) - } - display="block" - > - <EuiButtonIcon - aria-label={i18n.translate( - 'xpack.observabilityAiAssistant.chatFlyout.euiButtonIcon.expandConversationListLabel', - { defaultMessage: 'Expand conversation list' } - )} - className={expandButtonClassName} - color="text" - data-test-subj="observabilityAiAssistantChatFlyoutButton" - iconType={conversationsExpanded ? 'transitionLeftIn' : 'transitionLeftOut'} - onClick={() => setConversationsExpanded(!conversationsExpanded)} - /> - </EuiToolTip> - } - /> - - {conversationsExpanded ? ( - <ConversationList - selected={conversationId ?? ''} - onClickDeleteConversation={handleClickDeleteConversation} - onClickChat={handleClickChat} - onClickNewChat={handleClickNewChat} - /> - ) : ( + <EuiFlyout + closeButtonProps={{ + css: { marginRight: `${euiTheme.size.s}`, marginTop: `${euiTheme.size.s}` }, + }} + size={getFlyoutWidth({ + expanded: conversationsExpanded, + isSecondSlotVisible, + flyoutWidthMode, + })} + paddingSize="m" + onClose={() => { + onClose(); + setIsSecondSlotVisible(false); + if (secondSlotContainer) { + ReactDOM.unmountComponentAtNode(secondSlotContainer); + } + }} + > + <EuiFlexGroup gutterSize="none" className={containerClassName}> + <EuiFlexItem className={sidebarClass}> <EuiPopover anchorPosition="downLeft" + className={expandButtonContainerClassName} button={ <EuiToolTip - content={i18n.translate( - 'xpack.observabilityAiAssistant.chatFlyout.euiToolTip.newChatLabel', - { defaultMessage: 'New chat' } - )} + content={ + conversationsExpanded + ? i18n.translate( + 'xpack.observabilityAiAssistant.chatFlyout.euiToolTip.collapseConversationListLabel', + { defaultMessage: 'Collapse conversation list' } + ) + : i18n.translate( + 'xpack.observabilityAiAssistant.chatFlyout.euiToolTip.expandConversationListLabel', + { defaultMessage: 'Expand conversation list' } + ) + } display="block" > <EuiButtonIcon aria-label={i18n.translate( - 'xpack.observabilityAiAssistant.chatFlyout.euiButtonIcon.newChatLabel', - { defaultMessage: 'New chat' } + 'xpack.observabilityAiAssistant.chatFlyout.euiButtonIcon.expandConversationListLabel', + { defaultMessage: 'Expand conversation list' } )} - data-test-subj="observabilityAiAssistantNewChatFlyoutButton" - iconType="plusInCircle" - onClick={handleClickNewChat} + className={expandButtonClassName} + color="text" + data-test-subj="observabilityAiAssistantChatFlyoutButton" + iconType={conversationsExpanded ? 'transitionLeftIn' : 'transitionLeftOut'} + onClick={() => setConversationsExpanded(!conversationsExpanded)} /> </EuiToolTip> } - className={newChatButtonClassName} /> - )} - </EuiFlexItem> - <EuiFlexItem className={chatBodyContainerClassName}> - <ChatBody - key={chatBodyKeyRef.current} - chatFlyoutSecondSlotHandler={{ - container: secondSlotContainer, - setVisibility: setIsSecondSlotVisible, - }} - connectors={connectors} - currentUser={currentUser} - flyoutWidthMode={flyoutWidthMode} - initialTitle={initialTitle} - initialMessages={initialMessages} - initialConversationId={conversationId} - knowledgeBase={knowledgeBase} - showLinkToConversationsApp - startedFrom={startedFrom} - onConversationUpdate={(conversation) => { - setConversationId(conversation.conversation.id); - }} - onToggleFlyoutWidthMode={handleToggleFlyoutWidthMode} - /> - </EuiFlexItem> + {conversationsExpanded ? ( + <ConversationList + selected={conversationId ?? ''} + onClickDeleteConversation={handleClickDeleteConversation} + onClickChat={handleClickChat} + onClickNewChat={handleClickNewChat} + /> + ) : ( + <EuiPopover + anchorPosition="downLeft" + button={ + <EuiToolTip + content={i18n.translate( + 'xpack.observabilityAiAssistant.chatFlyout.euiToolTip.newChatLabel', + { defaultMessage: 'New chat' } + )} + display="block" + > + <EuiButtonIcon + aria-label={i18n.translate( + 'xpack.observabilityAiAssistant.chatFlyout.euiButtonIcon.newChatLabel', + { defaultMessage: 'New chat' } + )} + data-test-subj="observabilityAiAssistantNewChatFlyoutButton" + iconType="plusInCircle" + onClick={handleClickNewChat} + /> + </EuiToolTip> + } + className={newChatButtonClassName} + /> + )} + </EuiFlexItem> - <EuiFlexItem - style={{ - maxWidth: isSecondSlotVisible ? SIDEBAR_WIDTH : 0, - paddingTop: '56px', - }} - > - <ChatInlineEditingContent - setContainer={setSecondSlotContainer} - visible={isSecondSlotVisible} + <EuiFlexItem className={chatBodyContainerClassName}> + <ChatBody + key={chatBodyKeyRef.current} + connectors={connectors} + currentUser={currentUser} + flyoutWidthMode={flyoutWidthMode} + initialTitle={initialTitle} + initialMessages={initialMessages} + initialConversationId={conversationId} + knowledgeBase={knowledgeBase} + showLinkToConversationsApp + startedFrom={startedFrom} + onConversationUpdate={(conversation) => { + setConversationId(conversation.conversation.id); + }} + onToggleFlyoutWidthMode={handleToggleFlyoutWidthMode} + /> + </EuiFlexItem> + + <EuiFlexItem style={{ - borderTop: `solid 1px ${euiTheme.border.color}`, - borderLeft: `solid 1px ${euiTheme.border.color}`, + maxWidth: isSecondSlotVisible ? SIDEBAR_WIDTH : 0, + paddingTop: '56px', }} - /> - </EuiFlexItem> - </EuiFlexGroup> - </EuiFlyout> + > + <ChatInlineEditingContent + setContainer={setSecondSlotContainer} + visible={isSecondSlotVisible} + style={{ + borderTop: `solid 1px ${euiTheme.border.color}`, + borderLeft: `solid 1px ${euiTheme.border.color}`, + }} + /> + </EuiFlexItem> + </EuiFlexGroup> + </EuiFlyout> + </ObservabilityAIAssistantMultipaneFlyoutProvider> ) : null; } diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx index 48cf4070b0b96..0baccaf979f1f 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx @@ -13,7 +13,7 @@ import { omit } from 'lodash'; import type { Feedback } from '../feedback_buttons'; import type { Message } from '../../../common'; import type { UseKnowledgeBaseResult } from '../../hooks/use_knowledge_base'; -import type { ChatActionClickHandler, ChatFlyoutSecondSlotHandler } from './types'; +import type { ChatActionClickHandler } from './types'; import type { ObservabilityAIAssistantChatService } from '../../types'; import type { TelemetryEventTypeWithPayload } from '../../analytics'; import { ChatItem } from './chat_item'; @@ -54,7 +54,6 @@ export interface ChatTimelineProps { chatState: ChatState; currentUser?: Pick<AuthenticatedUser, 'full_name' | 'username'>; startedFrom?: StartedFrom; - chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler; onEdit: (message: Message, messageAfterEdit: Message) => void; onFeedback: (message: Message, feedback: Feedback) => void; onRegenerate: (message: Message) => void; @@ -69,7 +68,6 @@ export function ChatTimeline({ hasConnector, currentUser, startedFrom, - chatFlyoutSecondSlotHandler, onEdit, onFeedback, onRegenerate, @@ -86,7 +84,6 @@ export function ChatTimeline({ currentUser, startedFrom, chatState, - chatFlyoutSecondSlotHandler, onActionClick, }); @@ -110,16 +107,7 @@ export function ChatTimeline({ } return consolidatedChatItems; - }, [ - chatService, - hasConnector, - messages, - currentUser, - startedFrom, - chatState, - chatFlyoutSecondSlotHandler, - onActionClick, - ]); + }, [chatService, hasConnector, messages, currentUser, startedFrom, chatState, onActionClick]); return ( <EuiCommentList diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx index bf514691f7d93..0227ef42e2808 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx @@ -25,6 +25,7 @@ import { Disclaimer } from './disclaimer'; import { WelcomeMessageConnectors } from './welcome_message_connectors'; import { WelcomeMessageKnowledgeBase } from './welcome_message_knowledge_base'; import { useKibana } from '../../hooks/use_kibana'; +import { isSupportedConnectorType } from '../../../common/connectors'; const fullHeightClassName = css` height: 100%; @@ -68,7 +69,7 @@ export function WelcomeMessage({ const onConnectorCreated = (createdConnector: ActionConnector) => { setConnectorFlyoutOpen(false); - if (createdConnector.actionTypeId === '.gen-ai') { + if (isSupportedConnectorType(createdConnector.actionTypeId)) { connectors.reloadConnectors(); } diff --git a/x-pack/plugins/observability_ai_assistant/public/components/render_function.tsx b/x-pack/plugins/observability_ai_assistant/public/components/render_function.tsx index 91547deed0021..eded1c30e59e6 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/render_function.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/render_function.tsx @@ -7,27 +7,20 @@ import React from 'react'; import { Message } from '../../common'; import { useObservabilityAIAssistantChatService } from '../hooks/use_observability_ai_assistant_chat_service'; -import type { ChatActionClickHandler, ChatFlyoutSecondSlotHandler } from './chat/types'; +import type { ChatActionClickHandler } from './chat/types'; interface Props { name: string; arguments: string | undefined; response: Message['message']; onActionClick: ChatActionClickHandler; - chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler; } export function RenderFunction(props: Props) { const chatService = useObservabilityAIAssistantChatService(); return ( <> - {chatService.renderFunction( - props.name, - props.arguments, - props.response, - props.onActionClick, - props.chatFlyoutSecondSlotHandler - )} + {chatService.renderFunction(props.name, props.arguments, props.response, props.onActionClick)} </> ); } diff --git a/x-pack/plugins/observability_ai_assistant/public/context/observability_ai_assistant_multipane_flyout_provider.tsx b/x-pack/plugins/observability_ai_assistant/public/context/observability_ai_assistant_multipane_flyout_provider.tsx new file mode 100644 index 0000000000000..93a091ff4a7d4 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/public/context/observability_ai_assistant_multipane_flyout_provider.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createContext } from 'react'; +import type { ChatFlyoutSecondSlotHandler } from '../types'; + +export const ObservabilityAIAssistantMultipaneFlyoutContext = createContext< + ChatFlyoutSecondSlotHandler | undefined +>(undefined); + +export const ObservabilityAIAssistantMultipaneFlyoutProvider = + ObservabilityAIAssistantMultipaneFlyoutContext.Provider; diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx index 789697fbaaaa8..de7c4f04f241c 100644 --- a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx @@ -12,6 +12,7 @@ import type { LensPublicStart } from '@kbn/lens-plugin/public'; import { lensPluginMock } from '@kbn/lens-plugin/public/mocks/lens_plugin_mock'; import { uiActionsPluginMock } from '@kbn/ui-actions-plugin/public/mocks'; import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks'; +import { ObservabilityAIAssistantMultipaneFlyoutProvider } from '../context/observability_ai_assistant_multipane_flyout_provider'; import { VisualizeESQL } from './visualize_esql'; describe('VisualizeESQL', () => { @@ -50,19 +51,22 @@ describe('VisualizeESQL', () => { }, ] as DatatableColumn[]; render( - <VisualizeESQL - lens={lensService} - dataViews={dataViewsService} - uiActions={uiActionsService} - columns={columns} - query={'from foo | keep bytes, destination'} - onActionClick={jest.fn()} - userOverrides={userOverrides} - chatFlyoutSecondSlotHandler={{ + <ObservabilityAIAssistantMultipaneFlyoutProvider + value={{ container: document.createElement('div'), setVisibility: setVisibilitySpy ?? jest.fn(), }} - /> + > + <VisualizeESQL + lens={lensService} + dataViews={dataViewsService} + uiActions={uiActionsService} + columns={columns} + query={'from foo | keep bytes, destination'} + onActionClick={jest.fn()} + userOverrides={userOverrides} + /> + </ObservabilityAIAssistantMultipaneFlyoutProvider> ); } diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx index 05145c6130b4f..61295f4faf6f7 100644 --- a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx @@ -22,7 +22,7 @@ import type { TypedLensByValueInput, InlineEditLensEmbeddableContext, } from '@kbn/lens-plugin/public'; -import React, { useState, useEffect, useCallback, useMemo } from 'react'; +import React, { useState, useEffect, useCallback, useMemo, useContext } from 'react'; import ReactDOM from 'react-dom'; import useAsync from 'react-use/lib/useAsync'; import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils'; @@ -30,17 +30,14 @@ import { VisualizeESQLFunctionArguments, VisualizeESQLUserIntention, } from '../../common/functions/visualize_esql'; +import { ObservabilityAIAssistantMultipaneFlyoutContext } from '../context/observability_ai_assistant_multipane_flyout_provider'; import type { ObservabilityAIAssistantPluginStartDependencies, ObservabilityAIAssistantService, RegisterRenderFunctionDefinition, RenderFunction, } from '../types'; -import { - type ChatActionClickHandler, - ChatActionClickType, - ChatFlyoutSecondSlotHandler, -} from '../components/chat/types'; +import { type ChatActionClickHandler, ChatActionClickType } from '../components/chat/types'; interface VisualizeLensResponse { content: DatatableColumn[]; @@ -63,12 +60,6 @@ interface VisualizeESQLProps { * If not given, the embeddable gets them from the suggestions api */ userOverrides?: unknown; - /** Optional, should be passed if the embeddable is rendered in a flyout - * If not given, the inline editing push flyout won't open - * The code will be significantly improved, - * if this is addressed https://github.com/elastic/eui/issues/7443 - */ - chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler; /** User's preferation chart type as it comes from the model */ preferredChartType?: string; } @@ -85,7 +76,6 @@ export function VisualizeESQL({ query, onActionClick, userOverrides, - chatFlyoutSecondSlotHandler, preferredChartType, }: VisualizeESQLProps) { // fetch the pattern from the query @@ -100,6 +90,8 @@ export function VisualizeESQL({ }); }, [indexPattern]); + const chatFlyoutSecondSlotHandler = useContext(ObservabilityAIAssistantMultipaneFlyoutContext); + const [isSaveModalOpen, setIsSaveModalOpen] = useState(false); const [lensInput, setLensInput] = useState<TypedLensByValueInput | undefined>( userOverrides as TypedLensByValueInput @@ -316,7 +308,6 @@ export function registerVisualizeQueryRenderFunction({ arguments: { query, userOverrides, intention }, response, onActionClick, - chatFlyoutSecondSlotHandler, }: Parameters<RenderFunction<VisualizeESQLFunctionArguments, {}>>[0]) => { const { content } = response as VisualizeLensResponse; @@ -370,7 +361,6 @@ export function registerVisualizeQueryRenderFunction({ query={query} onActionClick={onActionClick} userOverrides={userOverrides} - chatFlyoutSecondSlotHandler={chatFlyoutSecondSlotHandler} preferredChartType={preferredChartType} /> ); diff --git a/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx b/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx index 4620c0cf2775d..0f49389c1c60d 100644 --- a/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx @@ -199,10 +199,6 @@ export function ConversationView() { showLinkToConversationsApp={false} startedFrom="conversationView" onConversationUpdate={handleConversationUpdate} - chatFlyoutSecondSlotHandler={{ - container: secondSlotContainer, - setVisibility: setIsSecondSlotVisible, - }} /> <div className={sidebarContainerClass}> diff --git a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts index 211f25b045b77..5fe933835eecd 100644 --- a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts +++ b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts @@ -9,8 +9,10 @@ import { AnalyticsServiceStart, HttpResponse } from '@kbn/core/public'; import { AbortError } from '@kbn/kibana-utils-plugin/common'; import { IncomingMessage } from 'http'; import { pick } from 'lodash'; -import { concatMap, delay, map, Observable, of, scan, shareReplay, timestamp } from 'rxjs'; +import { concatMap, delay, filter, map, Observable, of, scan, shareReplay, timestamp } from 'rxjs'; import { + BufferFlushEvent, + StreamingChatResponseEventType, StreamingChatResponseEventWithoutError, type StreamingChatResponseEvent, } from '../../common/conversation_complete'; @@ -116,7 +118,7 @@ export async function createChatService({ return { analytics, - renderFunction: (name, args, response, onActionClick, chatFlyoutSecondSlotHandler) => { + renderFunction: (name, args, response, onActionClick) => { const fn = renderFunctionRegistry.get(name); if (!fn) { @@ -134,7 +136,6 @@ export async function createChatService({ response: parsedResponse, arguments: parsedArguments, onActionClick, - chatFlyoutSecondSlotHandler, }); }, getContexts: () => contextDefinitions, @@ -164,7 +165,11 @@ export async function createChatService({ const response = _response as unknown as HttpResponse<IncomingMessage>; const response$ = toObservable(response) .pipe( - map((line) => JSON.parse(line) as StreamingChatResponseEvent), + map((line) => JSON.parse(line) as StreamingChatResponseEvent | BufferFlushEvent), + filter( + (line): line is StreamingChatResponseEvent => + line.type !== StreamingChatResponseEventType.BufferFlush + ), throwSerializedChatCompletionErrors() ) .subscribe(subscriber); @@ -225,7 +230,11 @@ export async function createChatService({ const subscription = toObservable(response) .pipe( - map((line) => JSON.parse(line) as StreamingChatResponseEvent), + map((line) => JSON.parse(line) as StreamingChatResponseEvent | BufferFlushEvent), + filter( + (line): line is StreamingChatResponseEvent => + line.type !== StreamingChatResponseEventType.BufferFlush + ), throwSerializedChatCompletionErrors() ) .subscribe(subscriber); diff --git a/x-pack/plugins/observability_ai_assistant/public/types.ts b/x-pack/plugins/observability_ai_assistant/public/types.ts index 418c7eca16b19..e303b01a5c9e9 100644 --- a/x-pack/plugins/observability_ai_assistant/public/types.ts +++ b/x-pack/plugins/observability_ai_assistant/public/types.ts @@ -49,6 +49,7 @@ import type { UseGenAIConnectorsResult } from './hooks/use_genai_connectors'; export type { CreateChatCompletionResponseChunk } from '../common/types'; export type { PendingMessage }; +export type { ChatFlyoutSecondSlotHandler }; export interface ObservabilityAIAssistantChatService { analytics: AnalyticsServiceStart; @@ -76,8 +77,7 @@ export interface ObservabilityAIAssistantChatService { name: string, args: string | undefined, response: { data?: string; content?: string }, - onActionClick: ChatActionClickHandler, - chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler + onActionClick: ChatActionClickHandler ) => React.ReactNode; } @@ -95,7 +95,6 @@ export type RenderFunction<TArguments, TResponse extends FunctionResponse> = (op arguments: TArguments; response: TResponse; onActionClick: ChatActionClickHandler; - chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler; }) => React.ReactNode; export type RegisterRenderFunctionDefinition< diff --git a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx index 600256d66a7bc..8135111a6f548 100644 --- a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx @@ -231,10 +231,6 @@ describe('getTimelineItemsFromConversation', () => { }, ], onActionClick: jest.fn(), - chatFlyoutSecondSlotHandler: { - container: null, - setVisibility: jest.fn(), - }, }); }); @@ -270,8 +266,7 @@ describe('getTimelineItemsFromConversation', () => { 'my_render_function', JSON.stringify({ foo: 'bar' }), { content: '[]', name: 'my_render_function', role: 'user' }, - expect.any(Function), - { container: null, setVisibility: expect.any(Function) } + expect.any(Function) ); expect(container.textContent).toEqual('Rendered'); diff --git a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx index 40b54708e5b6c..d1f14e30d6097 100644 --- a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx @@ -17,7 +17,7 @@ import { RenderFunction } from '../components/render_function'; import type { ObservabilityAIAssistantChatService } from '../types'; import { ChatState } from '../hooks/use_chat'; import { safeJsonParse } from './safe_json_parse'; -import type { ChatActionClickHandler, ChatFlyoutSecondSlotHandler } from '../components/chat/types'; +import type { ChatActionClickHandler } from '../components/chat/types'; function convertMessageToMarkdownCodeBlock(message: Message['message']) { let value: object; @@ -65,7 +65,6 @@ export function getTimelineItemsfromConversation({ messages, startedFrom, chatState, - chatFlyoutSecondSlotHandler, onActionClick, }: { chatService: ObservabilityAIAssistantChatService; @@ -74,7 +73,6 @@ export function getTimelineItemsfromConversation({ messages: Message[]; startedFrom?: StartedFrom; chatState: ChatState; - chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler; onActionClick: ChatActionClickHandler; }): ChatTimelineItem[] { const messagesWithoutSystem = messages.filter( @@ -169,7 +167,6 @@ export function getTimelineItemsfromConversation({ arguments={prevFunctionCall?.arguments} response={message.message} onActionClick={onActionClick} - chatFlyoutSecondSlotHandler={chatFlyoutSecondSlotHandler} /> ) : undefined; diff --git a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md index c5ff90ed582f2..76bf8a7fe7df2 100644 --- a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md +++ b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md @@ -26,7 +26,7 @@ By default, the tool will look for a Kibana instance running locally (at `http:/ #### Connector -Use `--connectorId` to specify a `.gen-ai` connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single `.gen-ai` connector is found, it will be used without prompting. +Use `--connectorId` to specify a `.gen-ai` or `.bedrock` connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single supported connector is found, it will be used without prompting. #### Persisting conversations diff --git a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts index d77e37a2b55a8..d0aa91f7ac53e 100644 --- a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts +++ b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts @@ -12,7 +12,9 @@ import { format, parse, UrlObject } from 'url'; import { ToolingLog } from '@kbn/tooling-log'; import pRetry from 'p-retry'; import { Message, MessageRole } from '../../common'; +import { isSupportedConnectorType } from '../../common/connectors'; import { + BufferFlushEvent, ChatCompletionChunkEvent, ChatCompletionErrorEvent, ConversationCreateEvent, @@ -217,7 +219,17 @@ export class KibanaClient { ) ).data ).pipe( - map((line) => JSON.parse(line) as ChatCompletionChunkEvent | ChatCompletionErrorEvent), + map( + (line) => + JSON.parse(line) as + | ChatCompletionChunkEvent + | ChatCompletionErrorEvent + | BufferFlushEvent + ), + filter( + (line): line is ChatCompletionChunkEvent | ChatCompletionErrorEvent => + line.type !== StreamingChatResponseEventType.BufferFlush + ), throwSerializedChatCompletionErrors(), concatenateChatCompletionChunks() ); @@ -270,13 +282,13 @@ export class KibanaClient { ) ).data ).pipe( - map((line) => JSON.parse(line) as StreamingChatResponseEvent), - throwSerializedChatCompletionErrors(), + map((line) => JSON.parse(line) as StreamingChatResponseEvent | BufferFlushEvent), filter( (event): event is MessageAddEvent | ConversationCreateEvent => event.type === StreamingChatResponseEventType.MessageAdd || event.type === StreamingChatResponseEventType.ConversationCreate ), + throwSerializedChatCompletionErrors(), toArray() ); @@ -427,6 +439,8 @@ export class KibanaClient { }) ); - return connectors.data.filter((connector) => connector.connector_type_id === '.gen-ai'); + return connectors.data.filter((connector) => + isSupportedConnectorType(connector.connector_type_id) + ); } } diff --git a/x-pack/plugins/observability_ai_assistant/server/functions/index.ts b/x-pack/plugins/observability_ai_assistant/server/functions/index.ts index d02f943c3523e..708f77da33321 100644 --- a/x-pack/plugins/observability_ai_assistant/server/functions/index.ts +++ b/x-pack/plugins/observability_ai_assistant/server/functions/index.ts @@ -53,7 +53,6 @@ export const registerFunctions: ChatRegistrationFunction = async ({ If multiple functions are suitable, use the most specific and easy one. E.g., when the user asks to visualise APM data, use the APM functions (if available) rather than "query". - Use the "get_dataset_info" function if it is not clear what fields or indices the user means, or if you want to get more information about the mappings. Note that ES|QL (the Elasticsearch query language, which is NOT Elasticsearch SQL, but a new piped language) is the preferred query language. @@ -66,6 +65,8 @@ export const registerFunctions: ChatRegistrationFunction = async ({ When the "visualize_query" function has been called, a visualization has been displayed to the user. DO NOT UNDER ANY CIRCUMSTANCES follow up a "visualize_query" function call with your own visualization attempt. If the "execute_query" function has been called, summarize these results for the user. The user does not see a visualization in this case. + Use the "get_dataset_info" function if it is not clear what fields or indices the user means, or if you want to get more information about the mappings. + If the "get_dataset_info" function returns no data, and the user asks for a query, generate a query anyway with the "query" function, but be explicit about it potentially being incorrect. ` ); diff --git a/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts b/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts index b69188d81b84a..86da0c0395587 100644 --- a/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts +++ b/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts @@ -113,6 +113,7 @@ export function registerQueryFunction({ type: 'boolean', }, }, + required: ['switch'], } as const, }, async ({ messages, connectorId }, signal) => { @@ -129,54 +130,58 @@ export function registerQueryFunction({ const source$ = ( await client.chat('classify_esql', { connectorId, - messages: withEsqlSystemMessage( - `Use the classify_esql function to classify the user's request - and get more information about specific functions and commands - you think are candidates for answering the question. - + messages: withEsqlSystemMessage().concat({ + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.User, + content: `Use the classify_esql function to classify the user's request + in the user message before this. + and get more information about specific functions and commands + you think are candidates for answering the question. - Examples for functions and commands: - Do you need to group data? Request \`STATS\`. - Extract data? Request \`DISSECT\` AND \`GROK\`. - Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`. - - For determining the intention of the user, the following options are available: - - ${VisualizeESQLUserIntention.generateQueryOnly}: the user only wants to generate the query, - but not run it. - - ${VisualizeESQLUserIntention.executeAndReturnResults}: the user wants to execute the query, - and have the assistant return/analyze/summarize the results. they don't need a - visualization. - - ${VisualizeESQLUserIntention.visualizeAuto}: The user wants to visualize the data from the - query, but wants us to pick the best visualization type, or their preferred - visualization is unclear. - - These intentions will display a specific visualization: - ${VisualizeESQLUserIntention.visualizeBar} - ${VisualizeESQLUserIntention.visualizeDonut} - ${VisualizeESQLUserIntention.visualizeHeatmap} - ${VisualizeESQLUserIntention.visualizeLine} - ${VisualizeESQLUserIntention.visualizeTagcloud} - ${VisualizeESQLUserIntention.visualizeTreemap} - ${VisualizeESQLUserIntention.visualizeWaffle} - ${VisualizeESQLUserIntention.visualizeXy} - - Some examples: - "Show me the avg of x" => ${VisualizeESQLUserIntention.executeAndReturnResults} - "Show me the results of y" => ${VisualizeESQLUserIntention.executeAndReturnResults} - "Display the sum of z" => ${VisualizeESQLUserIntention.executeAndReturnResults} - - "I want a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly} - "... Just show me the query" => ${VisualizeESQLUserIntention.generateQueryOnly} - "Create a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly} - - "Show me the avg of x over time" => ${VisualizeESQLUserIntention.visualizeAuto} - "I want a bar chart of ... " => ${VisualizeESQLUserIntention.visualizeBar} - "I want to see a heat map of ..." => ${VisualizeESQLUserIntention.visualizeHeatmap} - ` - ), + Examples for functions and commands: + Do you need to group data? Request \`STATS\`. + Extract data? Request \`DISSECT\` AND \`GROK\`. + Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`. + + For determining the intention of the user, the following options are available: + + ${VisualizeESQLUserIntention.generateQueryOnly}: the user only wants to generate the query, + but not run it. + + ${VisualizeESQLUserIntention.executeAndReturnResults}: the user wants to execute the query, + and have the assistant return/analyze/summarize the results. they don't need a + visualization. + + ${VisualizeESQLUserIntention.visualizeAuto}: The user wants to visualize the data from the + query, but wants us to pick the best visualization type, or their preferred + visualization is unclear. + + These intentions will display a specific visualization: + ${VisualizeESQLUserIntention.visualizeBar} + ${VisualizeESQLUserIntention.visualizeDonut} + ${VisualizeESQLUserIntention.visualizeHeatmap} + ${VisualizeESQLUserIntention.visualizeLine} + ${VisualizeESQLUserIntention.visualizeTagcloud} + ${VisualizeESQLUserIntention.visualizeTreemap} + ${VisualizeESQLUserIntention.visualizeWaffle} + ${VisualizeESQLUserIntention.visualizeXy} + + Some examples: + "Show me the avg of x" => ${VisualizeESQLUserIntention.executeAndReturnResults} + "Show me the results of y" => ${VisualizeESQLUserIntention.executeAndReturnResults} + "Display the sum of z" => ${VisualizeESQLUserIntention.executeAndReturnResults} + + "I want a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly} + "... Just show me the query" => ${VisualizeESQLUserIntention.generateQueryOnly} + "Create a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly} + + "Show me the avg of x over time" => ${VisualizeESQLUserIntention.visualizeAuto} + "I want a bar chart of ... " => ${VisualizeESQLUserIntention.visualizeBar} + "I want to see a heat map of ..." => ${VisualizeESQLUserIntention.visualizeHeatmap} + `, + }, + }), signal, functions: [ { @@ -184,6 +189,9 @@ export function registerQueryFunction({ description: `Use this function to determine: - what ES|QL functions and commands are candidates for answering the user's question - whether the user has requested a query, and if so, it they want it to be executed, or just shown. + + All parameters are required. Make sure the functions and commands you request are available in the + system message. `, parameters: { type: 'object', @@ -218,6 +226,10 @@ export function registerQueryFunction({ const response = await lastValueFrom(source$); + if (!response.message.function_call.arguments) { + throw new Error('LLM did not call classify_esql function'); + } + const args = JSON.parse(response.message.function_call.arguments) as { commands: string[]; functions: string[]; diff --git a/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts b/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts index 909a823286cc6..125c9a2f6eea0 100644 --- a/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts +++ b/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts @@ -11,11 +11,11 @@ import dedent from 'dedent'; import * as t from 'io-ts'; import { compact, last, omit } from 'lodash'; import { lastValueFrom } from 'rxjs'; +import { Logger } from '@kbn/logging'; import { FunctionRegistrationParameters } from '.'; import { MessageRole, type Message } from '../../common/types'; import { concatenateChatCompletionChunks } from '../../common/utils/concatenate_chat_completion_chunks'; import type { ObservabilityAIAssistantClient } from '../service/client'; -import { RespondFunctionResources } from '../service/types'; export function registerRecallFunction({ client, @@ -114,7 +114,7 @@ export function registerRecallFunction({ client, connectorId, signal, - resources, + logger: resources.logger, }); return { @@ -162,7 +162,7 @@ async function scoreSuggestions({ client, connectorId, signal, - resources, + logger, }: { suggestions: Awaited<ReturnType<typeof retrieveSuggestions>>; messages: Message[]; @@ -170,7 +170,7 @@ async function scoreSuggestions({ client: ObservabilityAIAssistantClient; connectorId: string; signal: AbortSignal; - resources: RespondFunctionResources; + logger: Logger; }) { const indexedSuggestions = suggestions.map((suggestion, index) => ({ ...suggestion, id: index })); @@ -233,6 +233,7 @@ async function scoreSuggestions({ }) ).pipe(concatenateChatCompletionChunks()) ); + const scoreFunctionRequest = decodeOrThrow(scoreFunctionRequestRt)(response); const { scores: scoresAsString } = decodeOrThrow(jsonRt.pipe(scoreFunctionArgumentsRt))( scoreFunctionRequest.message.function_call.arguments @@ -264,10 +265,7 @@ async function scoreSuggestions({ relevantDocumentIds.includes(suggestion.id) ); - resources.logger.debug( - `Found ${relevantDocumentIds.length} relevant suggestions from the knowledge base. ${scores.length} suggestions were considered in total.` - ); - resources.logger.debug(`Relevant documents: ${JSON.stringify(relevantDocuments, null, 2)}`); + logger.debug(`Relevant documents: ${JSON.stringify(relevantDocuments, null, 2)}`); return relevantDocuments; } diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts index 517cc48f9f27c..a9c58a9a59e00 100644 --- a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts +++ b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts @@ -5,13 +5,13 @@ * 2.0. */ import { notImplemented } from '@hapi/boom'; -import * as t from 'io-ts'; import { toBooleanRt } from '@kbn/io-ts-utils'; -import type OpenAI from 'openai'; +import * as t from 'io-ts'; import { Readable } from 'stream'; +import { flushBuffer } from '../../service/util/flush_buffer'; +import { observableIntoStream } from '../../service/util/observable_into_stream'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; import { messageRt } from '../runtime_types'; -import { observableIntoStream } from '../../service/util/observable_into_stream'; const chatRoute = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/chat', @@ -40,7 +40,10 @@ const chatRoute = createObservabilityAIAssistantServerRoute({ handler: async (resources): Promise<Readable> => { const { request, params, service } = resources; - const client = await service.getClient({ request }); + const [client, cloudStart] = await Promise.all([ + service.getClient({ request }), + resources.plugins.cloud?.start(), + ]); if (!client) { throw notImplemented(); @@ -68,7 +71,7 @@ const chatRoute = createObservabilityAIAssistantServerRoute({ : {}), }); - return observableIntoStream(response$); + return observableIntoStream(response$.pipe(flushBuffer(!!cloudStart?.isCloudEnabled))); }, }); @@ -90,10 +93,13 @@ const chatCompleteRoute = createObservabilityAIAssistantServerRoute({ }), ]), }), - handler: async (resources): Promise<Readable | OpenAI.Chat.ChatCompletion> => { + handler: async (resources): Promise<Readable> => { const { request, params, service } = resources; - const client = await service.getClient({ request }); + const [client, cloudStart] = await Promise.all([ + service.getClient({ request }), + resources.plugins.cloud?.start() || Promise.resolve(undefined), + ]); if (!client) { throw notImplemented(); @@ -125,7 +131,7 @@ const chatCompleteRoute = createObservabilityAIAssistantServerRoute({ functionClient, }); - return observableIntoStream(response$); + return observableIntoStream(response$.pipe(flushBuffer(!!cloudStart?.isCloudEnabled))); }, }); diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts index 894896fec6b3c..79134b9fef8d0 100644 --- a/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts +++ b/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts @@ -5,6 +5,7 @@ * 2.0. */ import { FindActionResult } from '@kbn/actions-plugin/server'; +import { isSupportedConnectorType } from '../../../common/connectors'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; const listConnectorsRoute = createObservabilityAIAssistantServerRoute({ @@ -21,7 +22,7 @@ const listConnectorsRoute = createObservabilityAIAssistantServerRoute({ const connectors = await actionsClient.getAll(); - return connectors.filter((connector) => connector.actionTypeId === '.gen-ai'); + return connectors.filter((connector) => isSupportedConnectorType(connector.actionTypeId)); }, }); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.test.ts new file mode 100644 index 0000000000000..e92d14088d337 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.test.ts @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Logger } from '@kbn/logging'; +import dedent from 'dedent'; +import { last } from 'lodash'; +import { MessageRole } from '../../../../common'; +import { createBedrockClaudeAdapter } from './bedrock_claude_adapter'; +import { LlmApiAdapterFactory } from './types'; + +describe('createBedrockClaudeAdapter', () => { + describe('getSubAction', () => { + function callSubActionFactory(overrides?: Partial<Parameters<LlmApiAdapterFactory>[0]>) { + const subActionParams = createBedrockClaudeAdapter({ + logger: { + debug: jest.fn(), + } as unknown as Logger, + functions: [ + { + name: 'my_tool', + description: 'My tool', + parameters: { + properties: { + myParam: { + type: 'string', + }, + }, + }, + }, + ], + messages: [ + { + '@timestamp': new Date().toString(), + message: { + role: MessageRole.System, + content: '', + }, + }, + { + '@timestamp': new Date().toString(), + message: { + role: MessageRole.User, + content: 'How can you help me?', + }, + }, + ], + ...overrides, + }).getSubAction().subActionParams as { + temperature: number; + messages: Array<{ role: string; content: string }>; + }; + + return { + ...subActionParams, + messages: subActionParams.messages.map((msg) => ({ ...msg, content: dedent(msg.content) })), + }; + } + describe('with functions', () => { + it('sets the temperature to 0', () => { + expect(callSubActionFactory().temperature).toEqual(0); + }); + + it('formats the functions', () => { + expect(callSubActionFactory().messages[0].content).toContain( + dedent(`<tools> + <tool_description> + <tool_name>my_tool</tool_name> + <description>My tool</description> + <parameters> + <parameter> + <name>myParam</name> + <type>string</type> + <description> + + Required: false + Multiple: false + + </description> + </parameter> + </parameters> + </tool_description> + </tools>`) + ); + }); + + it('replaces mentions of functions with tools', () => { + const messages = [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: + 'Call the "esql" tool. You can chain successive function calls, using the functions available.', + }, + }, + ]; + + const content = callSubActionFactory({ messages }).messages[0].content; + + expect(content).not.toContain(`"esql" function`); + expect(content).toContain(`"esql" tool`); + expect(content).not.toContain(`functions`); + expect(content).toContain(`tools`); + expect(content).toContain(`function calls`); + }); + + it('mentions to explicitly call the specified function if given', () => { + expect(last(callSubActionFactory({ functionCall: 'my_tool' }).messages)!.content).toContain( + 'Remember, use the my_tool tool to answer this question.' + ); + }); + + it('formats the function requests as XML', () => { + const messages = [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: '', + }, + }, + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.Assistant, + function_call: { + name: 'my_tool', + arguments: JSON.stringify({ myParam: 'myValue' }), + trigger: MessageRole.User as const, + }, + }, + }, + ]; + + expect(last(callSubActionFactory({ messages }).messages)!.content).toContain( + dedent(`<function_calls> + <invoke> + <tool_name>my_tool</tool_name> + <parameters> + <myParam>myValue</myParam> + </parameters> + </invoke> + </function_calls>`) + ); + }); + + it('formats errors', () => { + const messages = [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: '', + }, + }, + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.Assistant, + function_call: { + name: 'my_tool', + arguments: JSON.stringify({ myParam: 'myValue' }), + trigger: MessageRole.User as const, + }, + }, + }, + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.User, + name: 'my_tool', + content: JSON.stringify({ error: 'An internal server error occurred' }), + }, + }, + ]; + + expect(last(callSubActionFactory({ messages }).messages)!.content).toContain( + dedent(`<function_results> + <system> + <error>An internal server error occurred</error> + </system> + </function_results>`) + ); + }); + + it('formats function responses as XML + JSON', () => { + const messages = [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: '', + }, + }, + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.Assistant, + function_call: { + name: 'my_tool', + arguments: JSON.stringify({ myParam: 'myValue' }), + trigger: MessageRole.User as const, + }, + }, + }, + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.User, + name: 'my_tool', + content: JSON.stringify({ myResponse: { myParam: 'myValue' } }), + }, + }, + ]; + + expect(last(callSubActionFactory({ messages }).messages)!.content).toContain( + dedent(`<function_results> + <result> + <tool_name>my_tool</tool_name> + <stdout> + <myResponse> +<myParam>myValue</myParam> +</myResponse> + </stdout> + </result> + </function_results>`) + ); + }); + }); + }); + + describe('streamIntoObservable', () => { + // this data format is heavily encoded, so hard to reproduce. + // will leave this empty until we have some sample data. + }); +}); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.ts new file mode 100644 index 0000000000000..d5ba0d726ab12 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.ts @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import dedent from 'dedent'; +import { castArray } from 'lodash'; +import { filter, tap } from 'rxjs'; +import { Builder } from 'xml2js'; +import { createInternalServerError } from '../../../../common/conversation_complete'; +import { + BedrockChunkMember, + eventstreamSerdeIntoObservable, +} from '../../util/eventstream_serde_into_observable'; +import { jsonSchemaToFlatParameters } from '../../util/json_schema_to_flat_parameters'; +import { processBedrockStream } from './process_bedrock_stream'; +import type { LlmApiAdapterFactory } from './types'; + +function replaceFunctionsWithTools(content: string) { + return content.replaceAll(/(function)(s)?(?!\scall)/g, (match, p1, p2) => { + return `tool${p2 || ''}`; + }); +} + +// Most of the work here is to re-format OpenAI-compatible functions for Claude. +// See https://github.com/anthropics/anthropic-tools/blob/main/tool_use_package/prompt_constructors.py + +export const createBedrockClaudeAdapter: LlmApiAdapterFactory = ({ + messages, + functions, + functionCall, + logger, +}) => ({ + getSubAction: () => { + const [systemMessage, ...otherMessages] = messages; + + const filteredFunctions = functionCall + ? functions?.filter((fn) => fn.name === functionCall) + : functions; + + let functionsPrompt: string = ''; + + if (filteredFunctions?.length) { + functionsPrompt = `In this environment, you have access to a set of tools you can use to answer the user's question. + + When deciding what tool to use, keep in mind that you can call other tools in successive requests, so decide what tool + would be a good first step. + + You MUST only invoke a single tool, and invoke it once. Other invocations will be ignored. + You MUST wait for the results before invoking another. + You can call multiple tools in successive messages. This means you can chain function calls. If any tool was used in a previous + message, consider whether it still makes sense to follow it up with another function call. + + ${ + functions?.find((fn) => fn.name === 'recall') + ? `The "recall" function is ALWAYS used after a user question. Even if it was used before, your job is to answer the last user question, + even if the "recall" function was executed after that. Consider the tools you need to answer the user's question.` + : '' + } + + Rather than explaining how you would call a function, just generate the XML to call the function. It will automatically be + executed and returned to you. + + These results are generally not visible to the user. Treat them as if they are not, + unless specified otherwise. + + ONLY respond with XML, do not add any text. + + If a parameter allows multiple values, separate the values by "," + + You may call them like this. + + <function_calls> + <invoke> + <tool_name>$TOOL_NAME</tool_name> + <parameters> + <$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME> + ... + </parameters> + </invoke> + </function_calls> + + Here are the tools available: + + <tools> + ${filteredFunctions + .map( + (fn) => `<tool_description> + <tool_name>${fn.name}</tool_name> + <description>${fn.description}</description> + <parameters> + ${jsonSchemaToFlatParameters(fn.parameters).map((param) => { + return `<parameter> + <name>${param.name}</name> + <type>${param.type}</type> + <description> + ${param.description || ''} + Required: ${!!param.required} + Multiple: ${!!param.array} + ${ + param.enum || param.constant + ? `Allowed values: ${castArray(param.constant || param.enum).join(', ')}` + : '' + } + </description> + </parameter>`; + })} + </parameters> + </tool_description>` + ) + .join('\n')} + </tools> + + + Examples: + + Assistant: + <function_calls> + <invoke> + <tool_name>my_tool</tool_name> + <parameters> + <myParam>foo</myParam> + </parameters> + </invoke> + </function_calls> + + Assistant: + <function_calls> + <invoke> + <tool_name>another_tool</tool_name> + <parameters> + <myParam>foo</myParam> + </parameters> + </invoke> + </function_calls> + + `; + } + + const formattedMessages = [ + { + role: 'system', + content: `${replaceFunctionsWithTools(systemMessage.message.content!)} + + ${functionsPrompt} + `, + }, + ...otherMessages.map((message, index) => { + const builder = new Builder({ headless: true }); + if (message.message.name) { + const deserialized = JSON.parse(message.message.content || '{}'); + + if ('error' in deserialized) { + return { + role: message.message.role, + content: dedent(`<function_results> + <system> + ${builder.buildObject(deserialized)} + </system> + </function_results> + `), + }; + } + + return { + role: message.message.role, + content: dedent(` + <function_results> + <result> + <tool_name>${message.message.name}</tool_name> + <stdout> + ${builder.buildObject(deserialized)} + </stdout> + </result> + </function_results>`), + }; + } + + let content = replaceFunctionsWithTools(message.message.content || ''); + + if (message.message.function_call?.name) { + content += builder.buildObject({ + function_calls: { + invoke: { + tool_name: message.message.function_call.name, + parameters: JSON.parse(message.message.function_call.arguments || '{}'), + }, + }, + }); + } + + if (index === otherMessages.length - 1 && functionCall) { + content += ` + + Remember, use the ${functionCall} tool to answer this question.`; + } + + return { + role: message.message.role, + content, + }; + }), + ]; + + return { + subAction: 'invokeStream', + subActionParams: { + messages: formattedMessages, + temperature: 0, + stopSequences: ['\n\nHuman:', '</function_calls>'], + }, + }; + }, + streamIntoObservable: (readable) => + eventstreamSerdeIntoObservable(readable).pipe( + tap((value) => { + if ('modelStreamErrorException' in value) { + throw createInternalServerError(value.modelStreamErrorException.originalMessage); + } + }), + filter((value): value is BedrockChunkMember => { + return 'chunk' in value && value.chunk?.headers?.[':event-type']?.value === 'chunk'; + }), + processBedrockStream({ logger, functions }) + ), +}); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/openai_adapter.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/openai_adapter.ts new file mode 100644 index 0000000000000..61935d891a1db --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/openai_adapter.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { compact, isEmpty, omit } from 'lodash'; +import OpenAI from 'openai'; +import { MessageRole } from '../../../../common'; +import { processOpenAiStream } from '../../../../common/utils/process_openai_stream'; +import { eventsourceStreamIntoObservable } from '../../util/eventsource_stream_into_observable'; +import { LlmApiAdapterFactory } from './types'; + +export const createOpenAiAdapter: LlmApiAdapterFactory = ({ + messages, + functions, + functionCall, + logger, +}) => { + return { + getSubAction: () => { + const messagesForOpenAI: Array< + Omit<OpenAI.ChatCompletionMessageParam, 'role'> & { + role: MessageRole; + } + > = compact( + messages + .filter((message) => message.message.content || message.message.function_call?.name) + .map((message) => { + const role = + message.message.role === MessageRole.Elastic + ? MessageRole.User + : message.message.role; + + return { + role, + content: message.message.content, + function_call: isEmpty(message.message.function_call?.name) + ? undefined + : omit(message.message.function_call, 'trigger'), + name: message.message.name, + }; + }) + ); + + const functionsForOpenAI = functions; + + const request: Omit<OpenAI.ChatCompletionCreateParams, 'model'> & { model?: string } = { + messages: messagesForOpenAI as OpenAI.ChatCompletionCreateParams['messages'], + stream: true, + ...(!!functions?.length ? { functions: functionsForOpenAI } : {}), + temperature: 0, + function_call: functionCall ? { name: functionCall } : undefined, + }; + + return { + subAction: 'stream', + subActionParams: { + body: JSON.stringify(request), + stream: true, + }, + }; + }, + streamIntoObservable: (readable) => { + return eventsourceStreamIntoObservable(readable).pipe(processOpenAiStream()); + }, + }; +}; diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.test.ts new file mode 100644 index 0000000000000..78775b4d79d51 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.test.ts @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fromUtf8 } from '@smithy/util-utf8'; +import { lastValueFrom, of } from 'rxjs'; +import { Logger } from '@kbn/logging'; +import { concatenateChatCompletionChunks } from '../../../../common/utils/concatenate_chat_completion_chunks'; +import { processBedrockStream } from './process_bedrock_stream'; +import { MessageRole } from '../../../../common'; + +describe('processBedrockStream', () => { + const encode = (completion: string, stop?: string) => { + return { + chunk: { + headers: { + '::event-type': { value: 'chunk', type: 'uuid' as const }, + }, + body: fromUtf8( + JSON.stringify({ + bytes: Buffer.from(JSON.stringify({ completion, stop }), 'utf-8').toString('base64'), + }) + ), + }, + }; + }; + + function getLoggerMock() { + return { + debug: jest.fn(), + } as unknown as Logger; + } + + it('parses normal text messages', async () => { + expect( + await lastValueFrom( + of(encode('This'), encode(' is'), encode(' some normal'), encode(' text')).pipe( + processBedrockStream({ logger: getLoggerMock() }), + concatenateChatCompletionChunks() + ) + ) + ).toEqual({ + message: { + content: 'This is some normal text', + function_call: { + arguments: '', + name: '', + trigger: MessageRole.Assistant, + }, + role: MessageRole.Assistant, + }, + }); + }); + + it('parses function calls when no text is given', async () => { + expect( + await lastValueFrom( + of( + encode('<function_calls><invoke'), + encode('><tool_name>my_tool</tool_name><parameters'), + encode('><my_param>my_value</my_param'), + encode('></parameters></invoke'), + encode('>', '</function_calls>') + ).pipe( + processBedrockStream({ + logger: getLoggerMock(), + functions: [ + { + name: 'my_tool', + description: '', + parameters: { + properties: { + my_param: { + type: 'string', + }, + }, + }, + }, + ], + }), + concatenateChatCompletionChunks() + ) + ) + ).toEqual({ + message: { + content: '', + function_call: { + arguments: JSON.stringify({ my_param: 'my_value' }), + name: 'my_tool', + trigger: MessageRole.Assistant, + }, + role: MessageRole.Assistant, + }, + }); + }); + + it('parses function calls when they are prefaced by text', async () => { + expect( + await lastValueFrom( + of( + encode('This is'), + encode(' my text\n<function_calls><invoke'), + encode('><tool_name>my_tool</tool_name><parameters'), + encode('><my_param>my_value</my_param'), + encode('></parameters></invoke'), + encode('>', '</function_calls>') + ).pipe( + processBedrockStream({ + logger: getLoggerMock(), + functions: [ + { + name: 'my_tool', + description: '', + parameters: { + properties: { + my_param: { + type: 'string', + }, + }, + }, + }, + ], + }), + concatenateChatCompletionChunks() + ) + ) + ).toEqual({ + message: { + content: 'This is my text', + function_call: { + arguments: JSON.stringify({ my_param: 'my_value' }), + name: 'my_tool', + trigger: MessageRole.Assistant, + }, + role: MessageRole.Assistant, + }, + }); + }); + + it('throws an error if the XML cannot be parsed', async () => { + expect( + async () => + await lastValueFrom( + of( + encode('<function_calls><invoke'), + encode('><tool_name>my_tool</tool><parameters'), + encode('><my_param>my_value</my_param'), + encode('></parameters></invoke'), + encode('>', '</function_calls>') + ).pipe( + processBedrockStream({ + logger: getLoggerMock(), + functions: [ + { + name: 'my_tool', + description: '', + parameters: { + properties: { + my_param: { + type: 'string', + }, + }, + }, + }, + ], + }), + concatenateChatCompletionChunks() + ) + ) + ).rejects.toThrowErrorMatchingInlineSnapshot(` + "Unexpected close tag + Line: 0 + Column: 49 + Char: >" + `); + }); + + it('throws an error if the function does not exist', async () => { + expect( + async () => + await lastValueFrom( + of( + encode('<function_calls><invoke'), + encode('><tool_name>my_other_tool</tool_name><parameters'), + encode('><my_param>my_value</my_param'), + encode('></parameters></invoke'), + encode('>', '</function_calls>') + ).pipe( + processBedrockStream({ + logger: getLoggerMock(), + functions: [ + { + name: 'my_tool', + description: '', + parameters: { + properties: { + my_param: { + type: 'string', + }, + }, + }, + }, + ], + }), + concatenateChatCompletionChunks() + ) + ) + ).rejects.toThrowError( + 'Function definition for my_other_tool not found. Available are: my_tool' + ); + }); + + it('successfully invokes a function without parameters', async () => { + expect( + await lastValueFrom( + of( + encode('<function_calls><invoke'), + encode('><tool_name>my_tool</tool_name><parameters'), + encode('></parameters></invoke'), + encode('>', '</function_calls>') + ).pipe( + processBedrockStream({ + logger: getLoggerMock(), + functions: [ + { + name: 'my_tool', + description: '', + parameters: { + properties: { + my_param: { + type: 'string', + }, + }, + }, + }, + ], + }), + concatenateChatCompletionChunks() + ) + ) + ).toEqual({ + message: { + content: '', + function_call: { + arguments: '{}', + name: 'my_tool', + trigger: MessageRole.Assistant, + }, + role: MessageRole.Assistant, + }, + }); + }); +}); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.ts new file mode 100644 index 0000000000000..41bc19717485c --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { toUtf8 } from '@smithy/util-utf8'; +import { Observable } from 'rxjs'; +import { v4 } from 'uuid'; +import { Parser } from 'xml2js'; +import type { Logger } from '@kbn/logging'; +import { JSONSchema } from 'json-schema-to-ts'; +import { + ChatCompletionChunkEvent, + createInternalServerError, + StreamingChatResponseEventType, +} from '../../../../common/conversation_complete'; +import type { BedrockChunkMember } from '../../util/eventstream_serde_into_observable'; +import { convertDeserializedXmlWithJsonSchema } from '../../util/convert_deserialized_xml_with_json_schema'; + +async function parseFunctionCallXml({ + xml, + functions, +}: { + xml: string; + functions?: Array<{ name: string; description: string; parameters: JSONSchema }>; +}) { + const parser = new Parser(); + + const parsedValue = await parser.parseStringPromise(xml); + const invoke = parsedValue.function_calls.invoke[0]; + const fnName = invoke.tool_name[0]; + const parameters: Array<Record<string, string[]>> = invoke.parameters ?? []; + const functionDef = functions?.find((fn) => fn.name === fnName); + + if (!functionDef) { + throw createInternalServerError( + `Function definition for ${fnName} not found. ${ + functions?.length + ? 'Available are: ' + functions.map((fn) => fn.name).join(', ') + '.' + : 'No functions are available.' + }` + ); + } + + const args = convertDeserializedXmlWithJsonSchema(parameters, functionDef.parameters); + + return { + name: fnName, + arguments: JSON.stringify(args), + }; +} + +export function processBedrockStream({ + logger, + functions, +}: { + logger: Logger; + functions?: Array<{ name: string; description: string; parameters: JSONSchema }>; +}) { + return (source: Observable<BedrockChunkMember>) => + new Observable<ChatCompletionChunkEvent>((subscriber) => { + let functionCallsBuffer: string = ''; + const id = v4(); + + // We use this to make sure we don't complete the Observable + // before all operations have completed. + let nextPromise = Promise.resolve(); + + // As soon as we see a `<function` token, we write all chunks + // to a buffer, that we flush as a function request if we + // spot the stop sequence. + + async function handleNext(value: BedrockChunkMember) { + const response: { + completion: string; + stop_reason: string | null; + stop: null | string; + } = JSON.parse( + Buffer.from(JSON.parse(toUtf8(value.chunk.body)).bytes, 'base64').toString('utf-8') + ); + + let completion = response.completion; + + const isStartOfFunctionCall = !functionCallsBuffer && completion.includes('<function'); + + const isEndOfFunctionCall = functionCallsBuffer && response.stop === '</function_calls>'; + + const isInFunctionCall = !!functionCallsBuffer; + + if (isStartOfFunctionCall) { + const [before, after] = completion.split('<function'); + functionCallsBuffer += `<function${after}`; + completion = before.trimEnd(); + } else if (isEndOfFunctionCall) { + completion = ''; + functionCallsBuffer += response.completion + response.stop; + + logger.debug(`Parsing xml:\n${functionCallsBuffer}`); + + subscriber.next({ + id, + type: StreamingChatResponseEventType.ChatCompletionChunk, + message: { + content: '', + function_call: await parseFunctionCallXml({ + xml: functionCallsBuffer, + functions, + }), + }, + }); + + functionCallsBuffer = ''; + } else if (isInFunctionCall) { + completion = ''; + functionCallsBuffer += response.completion; + } + + if (completion.trim()) { + // OpenAI tokens come roughly separately, Bedrock/Claude + // chunks are bigger, so we split them up to give a more + // responsive feel in the UI + const parts = completion.split(' '); + parts.forEach((part, index) => { + subscriber.next({ + id, + type: StreamingChatResponseEventType.ChatCompletionChunk, + message: { + content: index === parts.length - 1 ? part : part + ' ', + }, + }); + }); + } + } + + source.subscribe({ + next: (value) => { + nextPromise = nextPromise.then(() => + handleNext(value).catch((error) => subscriber.error(error)) + ); + }, + error: (err) => { + subscriber.error(err); + }, + complete: () => { + nextPromise.then(() => subscriber.complete()); + }, + }); + }); +} diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/types.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/types.ts new file mode 100644 index 0000000000000..6ef3611bb4aae --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Readable } from 'node:stream'; +import type { Observable } from 'rxjs'; +import type { Logger } from '@kbn/logging'; +import type { Message } from '../../../../common'; +import type { ChatCompletionChunkEvent } from '../../../../common/conversation_complete'; +import type { CompatibleJSONSchema } from '../../../../common/types'; + +export type LlmApiAdapterFactory = (options: { + logger: Logger; + messages: Message[]; + functions?: Array<{ name: string; description: string; parameters: CompatibleJSONSchema }>; + functionCall?: string; +}) => LlmApiAdapter; + +export interface LlmApiAdapter { + getSubAction: () => { subAction: string; subActionParams: Record<string, any> }; + streamIntoObservable: (readable: Readable) => Observable<ChatCompletionChunkEvent>; +} diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts index fb22828247474..cbcbf0ea3fa3a 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts @@ -16,6 +16,7 @@ import { finished } from 'stream/promises'; import { ObservabilityAIAssistantClient } from '.'; import { createResourceNamesMap } from '..'; import { MessageRole, type Message } from '../../../common'; +import { ObservabilityAIAssistantConnectorType } from '../../../common/connectors'; import { ChatCompletionChunkEvent, ChatCompletionErrorCode, @@ -63,7 +64,7 @@ function createLlmSimulator() { ], }; await new Promise<void>((resolve, reject) => { - stream.write(`data: ${JSON.stringify(chunk)}\n`, undefined, (err) => { + stream.write(`data: ${JSON.stringify(chunk)}\n\n`, undefined, (err) => { return err ? reject(err) : resolve(); }); }); @@ -72,7 +73,7 @@ function createLlmSimulator() { if (stream.destroyed) { throw new Error('Stream is already destroyed'); } - await new Promise((resolve) => stream.write('data: [DONE]', () => stream.end(resolve))); + await new Promise((resolve) => stream.write('data: [DONE]\n\n', () => stream.end(resolve))); }, error: (error: Error) => { stream.destroy(error); @@ -85,6 +86,7 @@ describe('Observability AI Assistant client', () => { const actionsClientMock: DeeplyMockedKeys<ActionsClient> = { execute: jest.fn(), + get: jest.fn(), } as any; const internalUserEsClientMock: DeeplyMockedKeys<ElasticsearchClient> = { @@ -125,6 +127,15 @@ describe('Observability AI Assistant client', () => { return name !== 'recall'; }); + actionsClientMock.get.mockResolvedValue({ + actionTypeId: ObservabilityAIAssistantConnectorType.OpenAI, + id: 'foo', + name: 'My connector', + isPreconfigured: false, + isDeprecated: false, + isSystemAction: false, + }); + currentUserEsClientMock.search.mockResolvedValue({ hits: { hits: [], @@ -491,6 +502,8 @@ describe('Observability AI Assistant client', () => { stream.on('data', dataHandler); + await nextTick(); + await llmSimulator.next({ content: 'Hello' }); await llmSimulator.complete(); @@ -590,6 +603,8 @@ describe('Observability AI Assistant client', () => { stream.on('data', dataHandler); + await nextTick(); + await llmSimulator.next({ content: 'Hello' }); await new Promise((resolve) => @@ -598,7 +613,7 @@ describe('Observability AI Assistant client', () => { error: { message: 'Connection unexpectedly closed', }, - })}\n`, + })}\n\n`, resolve ) ); @@ -694,6 +709,8 @@ describe('Observability AI Assistant client', () => { stream.on('data', dataHandler); + await nextTick(); + await llmSimulator.next({ content: 'Hello', function_call: { name: 'my-function', arguments: JSON.stringify({ foo: 'bar' }) }, @@ -1259,6 +1276,8 @@ describe('Observability AI Assistant client', () => { await nextLlmCallPromise; } + await nextTick(); + await requestAlertsFunctionCall(); await requestAlertsFunctionCall(); @@ -1348,6 +1367,8 @@ describe('Observability AI Assistant client', () => { stream.on('data', dataHandler); + await nextTick(); + await llmSimulator.next({ function_call: { name: 'get_top_alerts' } }); await llmSimulator.complete(); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts index afd34aa8ea966..2fc6bb7be34cc 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts @@ -12,12 +12,11 @@ import type { Logger } from '@kbn/logging'; import type { PublicMethodsOf } from '@kbn/utility-types'; import apm from 'elastic-apm-node'; import { decode, encode } from 'gpt-tokenizer'; -import { compact, isEmpty, last, merge, noop, omit, pick, take } from 'lodash'; -import type OpenAI from 'openai'; +import { last, merge, noop, omit, pick, take } from 'lodash'; import { filter, - firstValueFrom, isObservable, + last as lastOperator, lastValueFrom, Observable, shareReplay, @@ -25,13 +24,14 @@ import { } from 'rxjs'; import { Readable } from 'stream'; import { v4 } from 'uuid'; +import { ObservabilityAIAssistantConnectorType } from '../../../common/connectors'; import { ChatCompletionChunkEvent, ChatCompletionErrorEvent, createConversationNotFoundError, + createTokenLimitReachedError, MessageAddEvent, StreamingChatResponseEventType, - createTokenLimitReachedError, type StreamingChatResponseEvent, } from '../../../common/conversation_complete'; import { @@ -47,7 +47,6 @@ import { } from '../../../common/types'; import { concatenateChatCompletionChunks } from '../../../common/utils/concatenate_chat_completion_chunks'; import { emitWithConcatenatedMessage } from '../../../common/utils/emit_with_concatenated_message'; -import { processOpenAiStream } from '../../../common/utils/process_openai_stream'; import type { ChatFunctionClient } from '../chat_function_client'; import { KnowledgeBaseEntryOperationType, @@ -56,7 +55,9 @@ import { } from '../knowledge_base_service'; import type { ObservabilityAIAssistantResourceNames } from '../types'; import { getAccessQuery } from '../util/get_access_query'; -import { streamIntoObservable } from '../util/stream_into_observable'; +import { createBedrockClaudeAdapter } from './adapters/bedrock_claude_adapter'; +import { createOpenAiAdapter } from './adapters/openai_adapter'; +import { LlmApiAdapter } from './adapters/types'; export class ObservabilityAIAssistantClient { constructor( @@ -465,111 +466,102 @@ export class ObservabilityAIAssistantClient { const spanId = (span?.ids['span.id'] || '').substring(0, 6); - const messagesForOpenAI: Array< - Omit<OpenAI.ChatCompletionMessageParam, 'role'> & { - role: MessageRole; - } - > = compact( - messages - .filter((message) => message.message.content || message.message.function_call?.name) - .map((message) => { - const role = - message.message.role === MessageRole.Elastic ? MessageRole.User : message.message.role; - - return { - role, - content: message.message.content, - function_call: isEmpty(message.message.function_call?.name) - ? undefined - : omit(message.message.function_call, 'trigger'), - name: message.message.name, - }; - }) - ); + try { + const connector = await this.dependencies.actionsClient.get({ + id: connectorId, + }); - const functionsForOpenAI = functions; + let adapter: LlmApiAdapter; + + switch (connector.actionTypeId) { + case ObservabilityAIAssistantConnectorType.OpenAI: + adapter = createOpenAiAdapter({ + logger: this.dependencies.logger, + messages, + functionCall, + functions, + }); + break; + + case ObservabilityAIAssistantConnectorType.Bedrock: + adapter = createBedrockClaudeAdapter({ + logger: this.dependencies.logger, + messages, + functionCall, + functions, + }); + break; + + default: + throw new Error(`Connector type is not supported: ${connector.actionTypeId}`); + } - const request: Omit<OpenAI.ChatCompletionCreateParams, 'model'> & { model?: string } = { - messages: messagesForOpenAI as OpenAI.ChatCompletionCreateParams['messages'], - stream: true, - ...(!!functions?.length ? { functions: functionsForOpenAI } : {}), - temperature: 0, - function_call: functionCall ? { name: functionCall } : undefined, - }; + const subAction = adapter.getSubAction(); - this.dependencies.logger.debug(`Sending conversation to connector`); - this.dependencies.logger.trace(JSON.stringify(request, null, 2)); + this.dependencies.logger.debug(`Sending conversation to connector`); + this.dependencies.logger.trace(JSON.stringify(subAction.subActionParams, null, 2)); - const now = performance.now(); + const now = performance.now(); - const executeResult = await this.dependencies.actionsClient.execute({ - actionId: connectorId, - params: { - subAction: 'stream', - subActionParams: { - body: JSON.stringify(request), - stream: true, - }, - }, - }); + const executeResult = await this.dependencies.actionsClient.execute({ + actionId: connectorId, + params: subAction, + }); - this.dependencies.logger.debug( - `Received action client response: ${executeResult.status} (took: ${Math.round( - performance.now() - now - )}ms)${spanId ? ` (${spanId})` : ''}` - ); + this.dependencies.logger.debug( + `Received action client response: ${executeResult.status} (took: ${Math.round( + performance.now() - now + )}ms)${spanId ? ` (${spanId})` : ''}` + ); - if (executeResult.status === 'error' && executeResult?.serviceMessage) { - const tokenLimitRegex = - /This model's maximum context length is (\d+) tokens\. However, your messages resulted in (\d+) tokens/g; - const tokenLimitRegexResult = tokenLimitRegex.exec(executeResult.serviceMessage); + if (executeResult.status === 'error' && executeResult?.serviceMessage) { + const tokenLimitRegex = + /This model's maximum context length is (\d+) tokens\. However, your messages resulted in (\d+) tokens/g; + const tokenLimitRegexResult = tokenLimitRegex.exec(executeResult.serviceMessage); - if (tokenLimitRegexResult) { - const [, tokenLimit, tokenCount] = tokenLimitRegexResult; - throw createTokenLimitReachedError(parseInt(tokenLimit, 10), parseInt(tokenCount, 10)); + if (tokenLimitRegexResult) { + const [, tokenLimit, tokenCount] = tokenLimitRegexResult; + throw createTokenLimitReachedError(parseInt(tokenLimit, 10), parseInt(tokenCount, 10)); + } } - } - if (executeResult.status === 'error') { - throw internal(`${executeResult?.message} - ${executeResult?.serviceMessage}`); - } + if (executeResult.status === 'error') { + throw internal(`${executeResult?.message} - ${executeResult?.serviceMessage}`); + } - const response = executeResult.data as Readable; + const response = executeResult.data as Readable; - signal.addEventListener('abort', () => response.destroy()); + signal.addEventListener('abort', () => response.destroy()); - const observable = streamIntoObservable(response).pipe(processOpenAiStream(), shareReplay()); + const response$ = adapter.streamIntoObservable(response).pipe(shareReplay()); - firstValueFrom(observable) - .catch(noop) - .finally(() => { - this.dependencies.logger.debug( - `Received first value after ${Math.round(performance.now() - now)}ms${ - spanId ? ` (${spanId})` : '' - }` - ); + response$.pipe(concatenateChatCompletionChunks(), lastOperator()).subscribe({ + error: (error) => { + this.dependencies.logger.debug('Error in chat response'); + this.dependencies.logger.debug(error); + }, + next: (message) => { + this.dependencies.logger.debug(`Received message:\n${JSON.stringify(message)}`); + }, }); - lastValueFrom(observable) - .then( - () => { + lastValueFrom(response$) + .then(() => { span?.setOutcome('success'); - }, - () => { + }) + .catch(() => { span?.setOutcome('failure'); - } - ) - .finally(() => { - this.dependencies.logger.debug( - `Completed response in ${Math.round(performance.now() - now)}ms${ - spanId ? ` (${spanId})` : '' - }` - ); - - span?.end(); - }); + }) + .finally(() => { + span?.end(); + }); - return observable; + return response$; + } catch (error) { + span?.setOutcome('failure'); + span?.end(); + throw error; + } }; find = async (options?: { query?: string }): Promise<{ conversations: Conversation[] }> => { @@ -631,13 +623,36 @@ export class ObservabilityAIAssistantClient { }) => { const response$ = await this.chat('generate_title', { messages: [ + { + '@timestamp': new Date().toString(), + message: { + role: MessageRole.System, + content: `You are a helpful assistant for Elastic Observability. Assume the following message is the start of a conversation between you and a user; give this conversation a title based on the content below. DO NOT UNDER ANY CIRCUMSTANCES wrap this title in single or double quotes. This title is shown in a list of conversations to the user, so title it for the user, not for you.`, + }, + }, { '@timestamp': new Date().toISOString(), message: { role: MessageRole.User, content: messages.slice(1).reduce((acc, curr) => { return `${acc} ${curr.message.role}: ${curr.message.content}`; - }, 'You are a helpful assistant for Elastic Observability. Assume the following message is the start of a conversation between you and a user; give this conversation a title based on the content below. DO NOT UNDER ANY CIRCUMSTANCES wrap this title in single or double quotes. This title is shown in a list of conversations to the user, so title it for the user, not for you. Here is the content:'), + }, 'Generate a title, using the title_conversation_function, based on the following conversation:\n\n'), + }, + }, + ], + functions: [ + { + name: 'title_conversation', + description: + 'Use this function to title the conversation. Do not wrap the title in quotes', + parameters: { + type: 'object', + properties: { + title: { + type: 'string', + }, + }, + required: ['title'], }, }, ], @@ -647,7 +662,10 @@ export class ObservabilityAIAssistantClient { const response = await lastValueFrom(response$.pipe(concatenateChatCompletionChunks())); - const input = response.message?.content || ''; + const input = + (response.message.function_call.name + ? JSON.parse(response.message.function_call.arguments).title + : response.message?.content) || ''; // This regular expression captures a string enclosed in single or double quotes. // It extracts the string content without the quotes. diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.test.ts new file mode 100644 index 0000000000000..8d1d64721abc4 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.test.ts @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { convertDeserializedXmlWithJsonSchema } from './convert_deserialized_xml_with_json_schema'; + +describe('deserializeXmlWithJsonSchema', () => { + it('deserializes XML into a JSON object according to the JSON schema', () => { + expect( + convertDeserializedXmlWithJsonSchema( + [ + { + foo: ['bar'], + }, + ], + { + type: 'object', + properties: { + foo: { + type: 'string', + }, + }, + } + ) + ).toEqual({ foo: 'bar' }); + }); + + it('converts strings to numbers if needed', () => { + expect( + convertDeserializedXmlWithJsonSchema( + [ + { + myNumber: ['0'], + }, + ], + { + type: 'object', + properties: { + myNumber: { + type: 'number', + }, + }, + } + ) + ).toEqual({ myNumber: 0 }); + }); + + it('de-dots object paths', () => { + expect( + convertDeserializedXmlWithJsonSchema( + [ + { + 'myObject.foo': ['bar'], + }, + ], + { + type: 'object', + properties: { + myObject: { + type: 'object', + properties: { + foo: { + type: 'string', + }, + }, + }, + }, + } + ) + ).toEqual({ + myObject: { + foo: 'bar', + }, + }); + }); + + it('casts to an array if needed', () => { + expect( + convertDeserializedXmlWithJsonSchema( + [ + { + myNumber: ['0'], + }, + ], + { + type: 'object', + properties: { + myNumber: { + type: 'number', + }, + }, + } + ) + ).toEqual({ + myNumber: 0, + }); + + expect( + convertDeserializedXmlWithJsonSchema( + [ + { + 'labels.myProp': ['myFirstValue, mySecondValue'], + }, + ], + { + type: 'object', + properties: { + labels: { + type: 'array', + items: { + type: 'object', + properties: { + myProp: { + type: 'string', + }, + }, + }, + }, + }, + } + ) + ).toEqual({ + labels: [{ myProp: 'myFirstValue' }, { myProp: 'mySecondValue' }], + }); + }); +}); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.ts new file mode 100644 index 0000000000000..a351edb9a33a1 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { set } from '@kbn/safer-lodash-set'; +import { unflatten } from 'flat'; +import type { JSONSchema } from 'json-schema-to-ts'; +import { forEach, get, isPlainObject } from 'lodash'; +import { jsonSchemaToFlatParameters } from './json_schema_to_flat_parameters'; + +// JS to XML is "lossy", e.g. everything becomes an array and a string, +// so we need a JSON schema to deserialize it + +export function convertDeserializedXmlWithJsonSchema( + parameterResults: Array<Record<string, string[]>>, + schema: JSONSchema +): Record<string, any> { + const parameters = jsonSchemaToFlatParameters(schema); + + const result: Record<string, any> = Object.fromEntries( + parameterResults.flatMap((parameterResult) => { + return Object.keys(parameterResult).map((name) => { + return [name, parameterResult[name]]; + }); + }) + ); + + parameters.forEach((param) => { + const key = param.name; + let value: any[] = result[key] ?? []; + value = param.array + ? String(value) + .split(',') + .map((val) => val.trim()) + : value; + + switch (param.type) { + case 'number': + value = value.map((val) => Number(val)); + break; + + case 'integer': + value = value.map((val) => Math.floor(Number(val))); + break; + + case 'boolean': + value = value.map((val) => String(val).toLowerCase() === 'true' || val === '1'); + break; + } + + result[key] = param.array ? value : value[0]; + }); + + function getArrayPaths(subSchema: JSONSchema, path: string = ''): string[] { + if (typeof subSchema === 'boolean') { + return []; + } + + if (subSchema.type === 'object') { + return Object.keys(subSchema.properties!).flatMap((key) => { + return getArrayPaths(subSchema.properties![key], path ? path + '.' + key : key); + }); + } + + if (subSchema.type === 'array') { + return [path, ...getArrayPaths(subSchema.items as JSONSchema, path)]; + } + + return []; + } + + const arrayPaths = getArrayPaths(schema); + + const unflattened: Record<string, any> = unflatten(result); + + arrayPaths.forEach((arrayPath) => { + const target: any[] = []; + function walk(value: any, path: string) { + if (Array.isArray(value)) { + value.forEach((val, index) => { + if (!target[index]) { + target[index] = {}; + } + if (path) { + set(target[index], path, val); + } else { + target[index] = val; + } + }); + } else if (isPlainObject(value)) { + forEach(value, (val, key) => { + walk(val, path ? path + '.' + key : key); + }); + } + } + const val = get(unflattened, arrayPath); + + walk(val, ''); + + set(unflattened, arrayPath, target); + }); + + return unflattened; +} diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/eventsource_stream_into_observable.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/eventsource_stream_into_observable.ts new file mode 100644 index 0000000000000..5ff332128f8ac --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/eventsource_stream_into_observable.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createParser } from 'eventsource-parser'; +import { Readable } from 'node:stream'; +import { Observable } from 'rxjs'; + +// OpenAI sends server-sent events, so we can use a library +// to deal with parsing, buffering, unicode etc + +export function eventsourceStreamIntoObservable(readable: Readable) { + return new Observable<string>((subscriber) => { + const parser = createParser((event) => { + if (event.type === 'event') { + subscriber.next(event.data); + } + }); + + async function processStream() { + for await (const chunk of readable) { + parser.feed(chunk.toString()); + } + } + + processStream().then( + () => { + subscriber.complete(); + }, + (error) => { + subscriber.error(error); + } + ); + }); +} diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/eventstream_serde_into_observable.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/eventstream_serde_into_observable.ts new file mode 100644 index 0000000000000..9252ec7588e3e --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/eventstream_serde_into_observable.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EventStreamMarshaller } from '@smithy/eventstream-serde-node'; +import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; +import { identity } from 'lodash'; +import { Observable } from 'rxjs'; +import { Readable } from 'stream'; +import { Message } from '@smithy/types'; + +interface ModelStreamErrorException { + name: 'ModelStreamErrorException'; + originalStatusCode?: number; + originalMessage?: string; +} + +export interface BedrockChunkMember { + chunk: Message; +} + +export interface ModelStreamErrorExceptionMember { + modelStreamErrorException: ModelStreamErrorException; +} + +export type BedrockStreamMember = BedrockChunkMember | ModelStreamErrorExceptionMember; + +// AWS uses SerDe to send over serialized data, so we use their +// @smithy library to parse the stream data + +export function eventstreamSerdeIntoObservable(readable: Readable) { + return new Observable<BedrockStreamMember>((subscriber) => { + const marshaller = new EventStreamMarshaller({ + utf8Encoder: toUtf8, + utf8Decoder: fromUtf8, + }); + + async function processStream() { + for await (const chunk of marshaller.deserialize(readable, identity)) { + if (chunk) { + subscriber.next(chunk as BedrockStreamMember); + } + } + } + + processStream().then( + () => { + subscriber.complete(); + }, + (error) => { + subscriber.error(error); + } + ); + }); +} diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/flush_buffer.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/flush_buffer.ts new file mode 100644 index 0000000000000..22723f1e49966 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/flush_buffer.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { repeat } from 'lodash'; +import { identity, Observable, OperatorFunction } from 'rxjs'; +import { + BufferFlushEvent, + StreamingChatResponseEventType, + StreamingChatResponseEventWithoutError, +} from '../../../common/conversation_complete'; + +// The Cloud proxy currently buffers 4kb or 8kb of data until flushing. +// This decreases the responsiveness of the streamed response, +// so we manually insert some data every 250ms if needed to force it +// to flush. + +export function flushBuffer<T extends StreamingChatResponseEventWithoutError>( + isCloud: boolean +): OperatorFunction<T, T | BufferFlushEvent> { + if (!isCloud) { + return identity; + } + + return (source: Observable<T>) => + new Observable<T | BufferFlushEvent>((subscriber) => { + const cloudProxyBufferSize = 4096; + let currentBufferSize: number = 0; + + const flushBufferIfNeeded = () => { + if (currentBufferSize && currentBufferSize <= cloudProxyBufferSize) { + subscriber.next({ + data: repeat('0', cloudProxyBufferSize * 2), + type: StreamingChatResponseEventType.BufferFlush, + }); + currentBufferSize = 0; + } + }; + + const intervalId = setInterval(flushBufferIfNeeded, 250); + + source.subscribe({ + next: (value) => { + currentBufferSize = + currentBufferSize <= cloudProxyBufferSize + ? JSON.stringify(value).length + currentBufferSize + : cloudProxyBufferSize; + subscriber.next(value); + }, + error: (error) => { + clearInterval(intervalId); + subscriber.error(error); + }, + complete: () => { + clearInterval(intervalId); + subscriber.complete(); + }, + }); + }); +} diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.test.ts new file mode 100644 index 0000000000000..afcfedf71dc85 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.test.ts @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { jsonSchemaToFlatParameters } from './json_schema_to_flat_parameters'; + +describe('jsonSchemaToFlatParameters', () => { + it('converts a simple object', () => { + expect( + jsonSchemaToFlatParameters({ + type: 'object', + properties: { + str: { + type: 'string', + }, + bool: { + type: 'boolean', + }, + }, + }) + ).toEqual([ + { + name: 'str', + type: 'string', + required: false, + }, + { + name: 'bool', + type: 'boolean', + required: false, + }, + ]); + }); + + it('handles descriptions', () => { + expect( + jsonSchemaToFlatParameters({ + type: 'object', + properties: { + str: { + type: 'string', + description: 'My string', + }, + }, + }) + ).toEqual([ + { + name: 'str', + type: 'string', + required: false, + description: 'My string', + }, + ]); + }); + + it('handles required properties', () => { + expect( + jsonSchemaToFlatParameters({ + type: 'object', + properties: { + str: { + type: 'string', + }, + bool: { + type: 'boolean', + }, + }, + required: ['str'], + }) + ).toEqual([ + { + name: 'str', + type: 'string', + required: true, + }, + { + name: 'bool', + type: 'boolean', + required: false, + }, + ]); + }); + + it('handles objects', () => { + expect( + jsonSchemaToFlatParameters({ + type: 'object', + properties: { + nested: { + type: 'object', + properties: { + str: { + type: 'string', + }, + }, + }, + }, + required: ['str'], + }) + ).toEqual([ + { + name: 'nested.str', + required: false, + type: 'string', + }, + ]); + }); + + it('handles arrays', () => { + expect( + jsonSchemaToFlatParameters({ + type: 'object', + properties: { + arr: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + required: ['str'], + }) + ).toEqual([ + { + name: 'arr', + required: false, + array: true, + type: 'string', + }, + ]); + + expect( + jsonSchemaToFlatParameters({ + type: 'object', + properties: { + arr: { + type: 'array', + items: { + type: 'object', + properties: { + foo: { + type: 'string', + }, + bar: { + type: 'object', + properties: { + baz: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + required: ['arr.foo.bar'], + }) + ).toEqual([ + { + name: 'arr.foo', + required: false, + array: true, + type: 'string', + }, + { + name: 'arr.bar.baz', + required: false, + array: true, + type: 'string', + }, + ]); + }); + + it('handles enum and const', () => { + expect( + jsonSchemaToFlatParameters({ + type: 'object', + properties: { + constant: { + type: 'string', + const: 'foo', + }, + enum: { + type: 'number', + enum: ['foo', 'bar'], + }, + }, + required: ['str'], + }) + ).toEqual([ + { + name: 'constant', + required: false, + type: 'string', + constant: 'foo', + }, + { + name: 'enum', + required: false, + type: 'number', + enum: ['foo', 'bar'], + }, + ]); + }); +}); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.ts new file mode 100644 index 0000000000000..cd984b0cfd7d0 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { JSONSchema } from 'json-schema-to-ts'; +import { castArray, isArray } from 'lodash'; + +interface Parameter { + name: string; + type: string; + description?: string; + required?: boolean; + enum?: unknown[]; + constant?: unknown; + array?: boolean; +} + +export function jsonSchemaToFlatParameters( + schema: JSONSchema, + name: string = '', + options: { required?: boolean; array?: boolean } = {} +): Parameter[] { + if (typeof schema === 'boolean') { + return []; + } + + switch (schema.type) { + case 'string': + case 'number': + case 'boolean': + case 'integer': + case 'null': + return [ + { + name, + type: schema.type, + description: schema.description, + array: options.array, + required: options.required, + constant: schema.const, + enum: schema.enum !== undefined ? castArray(schema.enum) : schema.enum, + }, + ]; + + case 'array': + if ( + typeof schema.items === 'boolean' || + typeof schema.items === 'undefined' || + isArray(schema.items) + ) { + return []; + } + return jsonSchemaToFlatParameters(schema.items as JSONSchema, name, { + ...options, + array: true, + }); + + default: + case 'object': + if (typeof schema.properties === 'undefined') { + return []; + } + return Object.entries(schema.properties).flatMap(([key, subSchema]) => { + return jsonSchemaToFlatParameters(subSchema, name ? `${name}.${key}` : key, { + ...options, + required: schema.required && schema.required.includes(key) ? true : false, + }); + }); + } +} diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts index a1ec52918453f..3ca09acde2b6f 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts @@ -8,14 +8,15 @@ import { Observable } from 'rxjs'; import { PassThrough } from 'stream'; import { + BufferFlushEvent, ChatCompletionErrorEvent, isChatCompletionError, - StreamingChatResponseEvent, StreamingChatResponseEventType, + StreamingChatResponseEventWithoutError, } from '../../../common/conversation_complete'; export function observableIntoStream( - source: Observable<Exclude<StreamingChatResponseEvent, ChatCompletionErrorEvent>> + source: Observable<StreamingChatResponseEventWithoutError | BufferFlushEvent> ) { const stream = new PassThrough(); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts index 764e39fdec152..b2c65c51da9cc 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts @@ -5,20 +5,25 @@ * 2.0. */ -import { concatMap, filter, from, map, Observable } from 'rxjs'; +import { Observable } from 'rxjs'; import type { Readable } from 'stream'; -export function streamIntoObservable(readable: Readable): Observable<string> { - let lineBuffer = ''; +export function streamIntoObservable(readable: Readable): Observable<any> { + return new Observable<string>((subscriber) => { + const decodedStream = readable; - return from(readable).pipe( - map((chunk: Buffer) => chunk.toString('utf-8')), - map((part) => { - const lines = (lineBuffer + part).split('\n'); - lineBuffer = lines.pop() || ''; // Keep the last incomplete line for the next chunk - return lines; - }), - concatMap((lines) => lines), - filter((line) => line.trim() !== '') - ); + async function processStream() { + for await (const chunk of decodedStream) { + subscriber.next(chunk); + } + } + + processStream() + .then(() => { + subscriber.complete(); + }) + .catch((error) => { + subscriber.error(error); + }); + }); } diff --git a/x-pack/plugins/observability_ai_assistant/server/types.ts b/x-pack/plugins/observability_ai_assistant/server/types.ts index ea2d3ee39e426..21fcc21f39a65 100644 --- a/x-pack/plugins/observability_ai_assistant/server/types.ts +++ b/x-pack/plugins/observability_ai_assistant/server/types.ts @@ -23,7 +23,8 @@ import type { } from '@kbn/data-views-plugin/server'; import type { MlPluginSetup, MlPluginStart } from '@kbn/ml-plugin/server'; import type { LicensingPluginSetup, LicensingPluginStart } from '@kbn/licensing-plugin/server'; -import { ObservabilityAIAssistantService } from './service'; +import type { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server'; +import type { ObservabilityAIAssistantService } from './service'; export interface ObservabilityAIAssistantPluginSetup { /** @@ -47,6 +48,7 @@ export interface ObservabilityAIAssistantPluginSetupDependencies { dataViews: DataViewsServerPluginSetup; ml: MlPluginSetup; licensing: LicensingPluginSetup; + cloud?: CloudSetup; } export interface ObservabilityAIAssistantPluginStartDependencies { actions: ActionsPluginStart; @@ -56,4 +58,5 @@ export interface ObservabilityAIAssistantPluginStartDependencies { dataViews: DataViewsServerPluginStart; ml: MlPluginStart; licensing: LicensingPluginStart; + cloud?: CloudStart; } diff --git a/x-pack/plugins/observability_ai_assistant/tsconfig.json b/x-pack/plugins/observability_ai_assistant/tsconfig.json index f5a29c470fe7a..13af731fd49db 100644 --- a/x-pack/plugins/observability_ai_assistant/tsconfig.json +++ b/x-pack/plugins/observability_ai_assistant/tsconfig.json @@ -61,6 +61,8 @@ "@kbn/apm-synthtrace-client", "@kbn/apm-synthtrace", "@kbn/code-editor", + "@kbn/safer-lodash-set", + "@kbn/cloud-plugin", "@kbn/ui-actions-plugin", "@kbn/expressions-plugin", "@kbn/visualization-utils", diff --git a/x-pack/plugins/observability_solution/logs_explorer/public/utils/convert_discover_app_state.ts b/x-pack/plugins/observability_solution/logs_explorer/public/utils/convert_discover_app_state.ts index 90d51f75e8c7c..639d4bdb2b0d1 100644 --- a/x-pack/plugins/observability_solution/logs_explorer/public/utils/convert_discover_app_state.ts +++ b/x-pack/plugins/observability_solution/logs_explorer/public/utils/convert_discover_app_state.ts @@ -16,7 +16,7 @@ import { GridColumnDisplayOptions, GridRowsDisplayOptions, } from '../../common'; -import { ControlOptions, OptionsListControlOption } from '../controller'; +import type { ControlOptions, OptionsListControl } from '../controller'; export const getGridColumnDisplayOptionsFromDiscoverAppState = ( discoverAppState: DiscoverAppState @@ -79,55 +79,78 @@ const createDiscoverPhrasesFilter = ({ key, values, negate, + index, }: { - values: PhraseFilterValue[]; + index: string; key: string; + values: PhraseFilterValue[]; negate?: boolean; -}): PhrasesFilter => - ({ - meta: { - key, - negate, - type: FILTERS.PHRASES, - params: values, - }, - query: { - bool: { - should: values.map((value) => ({ match_phrase: { [key]: value.toString() } })), - minimum_should_match: 1, - }, +}): PhrasesFilter => ({ + meta: { + index, + type: FILTERS.PHRASES, + key, + params: values.map((value) => value.toString()), + negate, + }, + query: { + bool: { + should: values.map((value) => ({ match_phrase: { [key]: value.toString() } })), + minimum_should_match: 1, }, - } as PhrasesFilter); + }, +}); const createDiscoverExistsFilter = ({ + index, key, negate, }: { key: string; + index: string; negate?: boolean; }): ExistsFilter => ({ meta: { + index, + type: FILTERS.EXISTS, + value: FILTERS.EXISTS, // Required for the filter to be displayed correctly in FilterBadge key, negate, - type: FILTERS.EXISTS, }, query: { exists: { field: key } }, }); -export const getDiscoverFiltersFromState = (filters: Filter[] = [], controls?: ControlOptions) => [ - ...filters, - ...(controls - ? (Object.keys(controls) as Array<keyof ControlOptions>).map((key) => - controls[key as keyof ControlOptions]?.selection.type === 'exists' - ? createDiscoverExistsFilter({ - key, - negate: controls[key]?.mode === 'exclude', - }) - : createDiscoverPhrasesFilter({ - key, - values: (controls[key]?.selection as OptionsListControlOption).selectedOptions, - negate: controls[key]?.mode === 'exclude', - }) - ) - : []), -]; +export const getDiscoverFiltersFromState = ( + index: string, + filters: Filter[] = [], + controls?: ControlOptions +) => { + return [ + ...filters, + ...(controls + ? (Object.entries(controls) as Array<[keyof ControlOptions, OptionsListControl]>).reduce< + Filter[] + >((acc, [key, control]) => { + if (control.selection.type === 'exists') { + acc.push( + createDiscoverExistsFilter({ + index, + key, + negate: control.mode === 'exclude', + }) + ); + } else if (control.selection.selectedOptions.length > 0) { + acc.push( + createDiscoverPhrasesFilter({ + index, + key, + values: control.selection.selectedOptions, + negate: control.mode === 'exclude', + }) + ); + } + return acc; + }, []) + : []), + ]; +}; diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/kibana.jsonc b/x-pack/plugins/observability_solution/observability_logs_explorer/kibana.jsonc index 42d762820aaad..8f6e248557efa 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/kibana.jsonc @@ -23,7 +23,12 @@ "datasetQuality" ], "optionalPlugins": [ - "serverless" + "serverless", + "triggersActionsUi", + "unifiedSearch", + "dataViews", + "dataViewEditor", + "lens" ], "requiredBundles": [ "kibanaReact" diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/public/applications/observability_logs_explorer.tsx b/x-pack/plugins/observability_solution/observability_logs_explorer/public/applications/observability_logs_explorer.tsx index 5f6739a5dfe3d..cab3742c06f05 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/public/applications/observability_logs_explorer.tsx +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/public/applications/observability_logs_explorer.tsx @@ -6,6 +6,7 @@ */ import { CoreStart } from '@kbn/core/public'; +import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common'; import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { Route, Router, Routes } from '@kbn/shared-ux-router'; import React from 'react'; @@ -57,6 +58,7 @@ export const ObservabilityLogsExplorerApp = ({ plugins, pluginStart, }: ObservabilityLogsExplorerAppProps) => { + const isDarkMode = core.theme.getTheme().darkMode; const KibanaContextProviderForPlugin = useKibanaContextForPluginProvider( core, plugins, @@ -69,10 +71,20 @@ export const ObservabilityLogsExplorerApp = ({ <KibanaContextProviderForPlugin> <KbnUrlStateStorageFromRouterProvider> <Router history={appParams.history}> - <Routes> - <Route path="/" exact={true} render={() => <ObservabilityLogsExplorerMainRoute />} /> - <Route path="/dataset-quality" exact={true} render={() => <DatasetQualityRoute />} /> - </Routes> + <EuiThemeProvider darkMode={isDarkMode}> + <Routes> + <Route + path="/" + exact={true} + render={() => <ObservabilityLogsExplorerMainRoute />} + /> + <Route + path="/dataset-quality" + exact={true} + render={() => <DatasetQualityRoute />} + /> + </Routes> + </EuiThemeProvider> </Router> </KbnUrlStateStorageFromRouterProvider> </KibanaContextProviderForPlugin> diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/alerts_popover.tsx b/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/alerts_popover.tsx new file mode 100644 index 0000000000000..22f689010a2df --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/alerts_popover.tsx @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiPopover, EuiButtonEmpty, EuiContextMenuPanel, EuiContextMenuItem } from '@elastic/eui'; +import React, { useMemo, useReducer } from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils'; +import { useActor } from '@xstate/react'; +import { hydrateDatasetSelection } from '@kbn/logs-explorer-plugin/common'; +import { getDiscoverFiltersFromState } from '@kbn/logs-explorer-plugin/public'; +import type { AlertParams } from '@kbn/observability-plugin/public/components/custom_threshold/types'; +import { useLinkProps } from '@kbn/observability-shared-plugin/public'; +import { useKibanaContextForPlugin } from '../utils/use_kibana'; +import { useObservabilityLogsExplorerPageStateContext } from '../state_machines/observability_logs_explorer/src'; + +type ThresholdRuleTypeParams = Pick<AlertParams, 'searchConfiguration'>; + +interface AlertsPopoverState { + isPopoverOpen: boolean; + isAddRuleFlyoutOpen: boolean; +} + +type AlertsPopoverAction = + | { + type: 'togglePopover'; + isOpen?: boolean; + } + | { + type: 'toggleAddRuleFlyout'; + isOpen?: boolean; + }; + +function alertsPopoverReducer(state: AlertsPopoverState, action: AlertsPopoverAction) { + switch (action.type) { + case 'togglePopover': + return { + isPopoverOpen: action.isOpen ?? !state.isPopoverOpen, + isAddRuleFlyoutOpen: state.isAddRuleFlyoutOpen, + }; + + case 'toggleAddRuleFlyout': + return { + isPopoverOpen: false, + isAddRuleFlyoutOpen: action.isOpen ?? !state.isAddRuleFlyoutOpen, + }; + + default: + return state; + } +} + +export const AlertsPopover = () => { + const { + services: { triggersActionsUi }, + } = useKibanaContextForPlugin(); + + const manageRulesLinkProps = useLinkProps({ app: 'observability', pathname: '/alerts/rules' }); + + const [pageState] = useActor(useObservabilityLogsExplorerPageStateContext()); + + const [state, dispatch] = useReducer(alertsPopoverReducer, { + isPopoverOpen: false, + isAddRuleFlyoutOpen: false, + }); + + const togglePopover = () => dispatch({ type: 'togglePopover' }); + const closePopover = () => dispatch({ type: 'togglePopover', isOpen: false }); + const openAddRuleFlyout = () => dispatch({ type: 'toggleAddRuleFlyout', isOpen: true }); + const closeAddRuleFlyout = () => dispatch({ type: 'toggleAddRuleFlyout', isOpen: false }); + + const addRuleFlyout = useMemo(() => { + if ( + state.isAddRuleFlyoutOpen && + triggersActionsUi && + pageState.matches({ initialized: 'validLogsExplorerState' }) + ) { + const { logsExplorerState } = pageState.context; + const index = hydrateDatasetSelection(logsExplorerState.datasetSelection).toDataviewSpec(); + + return triggersActionsUi.getAddRuleFlyout<ThresholdRuleTypeParams>({ + consumer: 'logs', + ruleTypeId: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID, + canChangeTrigger: false, + initialValues: { + params: { + searchConfiguration: { + index, + query: logsExplorerState.query, + filter: getDiscoverFiltersFromState( + index.id, + logsExplorerState.filters, + logsExplorerState.controls + ), + }, + }, + }, + onClose: closeAddRuleFlyout, + }); + } + }, [triggersActionsUi, pageState, state.isAddRuleFlyoutOpen]); + + return ( + <> + {state.isAddRuleFlyoutOpen && addRuleFlyout} + <EuiPopover + button={ + <EuiButtonEmpty onClick={togglePopover} iconType="arrowDown" iconSide="right"> + <FormattedMessage + id="xpack.observabilityLogsExplorer.alertsPopover.buttonLabel" + defaultMessage="Alerts" + /> + </EuiButtonEmpty> + } + isOpen={state.isPopoverOpen} + closePopover={closePopover} + panelPaddingSize="none" + anchorPosition="downLeft" + > + <EuiContextMenuPanel + items={[ + <EuiContextMenuItem key="createRule" icon="bell" onClick={openAddRuleFlyout}> + <FormattedMessage + id="xpack.observabilityLogsExplorer.alertsPopover.createRuleMenuItem" + defaultMessage="Create rule" + /> + </EuiContextMenuItem>, + <EuiContextMenuItem key="manageRules" icon="tableOfContents" {...manageRulesLinkProps}> + <FormattedMessage + id="xpack.observabilityLogsExplorer.alertsPopover.manageRulesMenuItem" + defaultMessage="Manage rules" + /> + </EuiContextMenuItem>, + ]} + /> + </EuiPopover> + </> + ); +}; diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/discover_link.tsx b/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/discover_link.tsx index b12390cc952a1..7c0b4596b4326 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/discover_link.tsx +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/discover_link.tsx @@ -53,18 +53,22 @@ export const DiscoverLinkForValidState = React.memo( discover: DiscoverStart; pageState: InitializedPageState; }) => { - const discoverLinkParams = useMemo<DiscoverAppLocatorParams>( - () => ({ + const discoverLinkParams = useMemo<DiscoverAppLocatorParams>(() => { + const index = hydrateDatasetSelection(logsExplorerState.datasetSelection).toDataviewSpec(); + return { breakdownField: logsExplorerState.chart.breakdownField ?? undefined, columns: getDiscoverColumnsFromDisplayOptions(logsExplorerState), - filters: getDiscoverFiltersFromState(logsExplorerState.filters, logsExplorerState.controls), + filters: getDiscoverFiltersFromState( + index.id, + logsExplorerState.filters, + logsExplorerState.controls + ), query: logsExplorerState.query, refreshInterval: logsExplorerState.refreshInterval, timeRange: logsExplorerState.time, - dataViewSpec: hydrateDatasetSelection(logsExplorerState.datasetSelection).toDataviewSpec(), - }), - [logsExplorerState] - ); + dataViewSpec: index, + }; + }, [logsExplorerState]); return <DiscoverLink discover={discover} discoverLinkParams={discoverLinkParams} />; } diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/logs_explorer_top_nav_menu.tsx b/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/logs_explorer_top_nav_menu.tsx index 9c2ea0a5e4817..0f64f586ab3fd 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/logs_explorer_top_nav_menu.tsx +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/public/components/logs_explorer_top_nav_menu.tsx @@ -26,6 +26,7 @@ import { useKibanaContextForPlugin } from '../utils/use_kibana'; import { ConnectedDiscoverLink } from './discover_link'; import { FeedbackLink } from './feedback_link'; import { ConnectedOnboardingLink } from './onboarding_link'; +import { AlertsPopover } from './alerts_popover'; export const LogsExplorerTopNavMenu = () => { const { @@ -67,6 +68,8 @@ const ServerlessTopNav = () => { <VerticalRule /> <FeedbackLink /> <VerticalRule /> + <AlertsPopover /> + <VerticalRule /> {ObservabilityAIAssistantActionMenuItem ? ( <ObservabilityAIAssistantActionMenuItem /> ) : null} @@ -143,6 +146,8 @@ const StatefulTopNav = () => { <EuiHeaderLinks gutterSize="xs"> <ConnectedDiscoverLink /> <VerticalRule /> + <AlertsPopover /> + <VerticalRule /> {ObservabilityAIAssistantActionMenuItem ? ( <ObservabilityAIAssistantActionMenuItem /> ) : null} diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/public/types.ts b/x-pack/plugins/observability_solution/observability_logs_explorer/public/types.ts index c3f094033f697..96754cfdab021 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/public/types.ts +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/public/types.ts @@ -15,6 +15,11 @@ import { AppMountParameters, ScopedHistory } from '@kbn/core/public'; import { LogsSharedClientStartExports } from '@kbn/logs-shared-plugin/public'; import { DatasetQualityPluginStart } from '@kbn/dataset-quality-plugin/public'; import { ObservabilityAIAssistantPluginStart } from '@kbn/observability-ai-assistant-plugin/public'; +import { TriggersAndActionsUIPublicPluginStart } from '@kbn/triggers-actions-ui-plugin/public'; +import { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; +import { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; +import { DataViewEditorStart } from '@kbn/data-view-editor-plugin/public'; +import { LensPublicStart } from '@kbn/lens-plugin/public'; import { ObservabilityLogsExplorerLocators, ObservabilityLogsExplorerLocationState, @@ -41,6 +46,11 @@ export interface ObservabilityLogsExplorerStartDeps { observabilityAIAssistant: ObservabilityAIAssistantPluginStart; observabilityShared: ObservabilitySharedPluginStart; serverless?: ServerlessPluginStart; + triggersActionsUi?: TriggersAndActionsUIPublicPluginStart; + unifiedSearch?: UnifiedSearchPublicPluginStart; + dataViews?: DataViewsPublicPluginStart; + dataViewEditor?: DataViewEditorStart; + lens?: LensPublicStart; share: SharePluginStart; datasetQuality: DatasetQualityPluginStart; } diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/tsconfig.json b/x-pack/plugins/observability_solution/observability_logs_explorer/tsconfig.json index 7192e3001a70b..c434a418c4246 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/tsconfig.json @@ -38,6 +38,13 @@ "@kbn/xstate-utils", "@kbn/router-utils", "@kbn/observability-ai-assistant-plugin", + "@kbn/rule-data-utils", + "@kbn/observability-plugin", + "@kbn/triggers-actions-ui-plugin", + "@kbn/unified-search-plugin", + "@kbn/data-views-plugin", + "@kbn/data-view-editor-plugin", + "@kbn/lens-plugin", ], "exclude": [ "target/**/*" diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx index 1e0c7021929c9..05de5e9725399 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx @@ -67,11 +67,27 @@ const platinumBaseColumns = [ initialWidth: 450, }, { columnHeaderType: 'not-filtered', id: 'host.name' }, - { columnHeaderType: 'not-filtered', id: 'host.risk.calculated_level' }, { columnHeaderType: 'not-filtered', id: 'user.name' }, - { columnHeaderType: 'not-filtered', id: 'user.risk.calculated_level' }, - { columnHeaderType: 'not-filtered', id: 'host.asset.criticality' }, - { columnHeaderType: 'not-filtered', id: 'user.asset.criticality' }, + { + columnHeaderType: 'not-filtered', + id: 'host.risk.calculated_level', + displayAsText: 'Host Risk Level', + }, + { + columnHeaderType: 'not-filtered', + id: 'user.risk.calculated_level', + displayAsText: 'User Risk Level', + }, + { + columnHeaderType: 'not-filtered', + id: 'host.asset.criticality', + displayAsText: 'Host Criticality', + }, + { + columnHeaderType: 'not-filtered', + id: 'user.asset.criticality', + displayAsText: 'User Criticality', + }, { columnHeaderType: 'not-filtered', id: 'process.name' }, { columnHeaderType: 'not-filtered', id: 'file.name' }, { columnHeaderType: 'not-filtered', id: 'source.ip' }, diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index ae0c05d2bdb05..4a7ea8e77cc92 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -137,6 +137,34 @@ export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate( } ); +export const ALERTS_HEADERS_HOST_RISK_LEVEL = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.hostRiskLevel', + { + defaultMessage: 'Host Risk Level', + } +); + +export const ALERTS_HEADERS_USER_RISK_LEVEL = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.userRiskLevel', + { + defaultMessage: 'User Risk Level', + } +); + +export const ALERTS_HEADERS_HOST_CRITICALITY = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.hostCriticality', + { + defaultMessage: 'Host Criticality', + } +); + +export const ALERTS_HEADERS_USER_CRITICALITY = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.userCriticality', + { + defaultMessage: 'User Criticality', + } +); + export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate( 'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle', { diff --git a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts index fc3f5afa897a2..6ca0a67244179 100644 --- a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts +++ b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts @@ -34,6 +34,18 @@ export const assigneesColumn: ColumnHeaderOptions = { initialWidth: DEFAULT_DATE_COLUMN_MIN_WIDTH, }; +export const hostRiskLevelColumn: ColumnHeaderOptions = { + columnHeaderType: defaultColumnHeaderType, + id: ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, + displayAsText: i18n.ALERTS_HEADERS_HOST_RISK_LEVEL, +}; + +export const userRiskLevelColumn: ColumnHeaderOptions = { + columnHeaderType: defaultColumnHeaderType, + id: ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, + displayAsText: i18n.ALERTS_HEADERS_USER_RISK_LEVEL, +}; + const getBaseColumns = ( license?: LicenseService ): Array< @@ -63,32 +75,24 @@ const getBaseColumns = ( columnHeaderType: defaultColumnHeaderType, id: 'host.name', }, - isPlatinumPlus - ? { - columnHeaderType: defaultColumnHeaderType, - id: ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL, - } - : null, { columnHeaderType: defaultColumnHeaderType, id: 'user.name', }, - isPlatinumPlus - ? { - columnHeaderType: defaultColumnHeaderType, - id: ALERT_USER_RISK_SCORE_CALCULATED_LEVEL, - } - : null, + isPlatinumPlus ? hostRiskLevelColumn : null, + isPlatinumPlus ? userRiskLevelColumn : null, isPlatinumPlus ? { columnHeaderType: defaultColumnHeaderType, id: ALERT_HOST_CRITICALITY, + displayAsText: i18n.ALERTS_HEADERS_HOST_CRITICALITY, } : null, isPlatinumPlus ? { columnHeaderType: defaultColumnHeaderType, id: ALERT_USER_CRITICALITY, + displayAsText: i18n.ALERTS_HEADERS_USER_CRITICALITY, } : null, { diff --git a/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx b/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx index 003d693a0182e..2fa32b82b8767 100644 --- a/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx +++ b/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx @@ -11,7 +11,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { CRITICALITY_LEVEL_TITLE, CRITICALITY_LEVEL_DESCRIPTION } from './translations'; import type { CriticalityLevel } from '../../../../common/entity_analytics/asset_criticality/types'; -const CRITICALITY_LEVEL_COLOR: Record<CriticalityLevel, string> = { +export const CRITICALITY_LEVEL_COLOR: Record<CriticalityLevel, string> = { very_important: '#E7664C', important: '#D6BF57', normal: '#54B399', diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.test.tsx new file mode 100644 index 0000000000000..2a52c4509492a --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.test.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TestProviders } from '../../../../../common/mock'; +import { render } from '@testing-library/react'; +import React from 'react'; +import { AssetCriticalityLevel } from './asset_criticality_level'; + +jest.mock('../../../../../common/components/draggables', () => ({ + DefaultDraggable: ({ children }: { children: React.ReactNode }) => <>{children}</>, +})); + +const defaultProps = { + contextId: 'testContext', + eventId: 'testEvent', + fieldName: 'testField', + fieldType: 'testType', + isAggregatable: true, + isDraggable: true, + value: 'low', +}; + +describe('AssetCriticalityLevel', () => { + it('renders', () => { + const { getByTestId } = render(<AssetCriticalityLevel {...defaultProps} />, { + wrapper: TestProviders, + }); + + expect(getByTestId('AssetCriticalityLevel-score-badge')).toHaveTextContent('low'); + }); +}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.tsx new file mode 100644 index 0000000000000..ac3fa8d977b61 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.tsx @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiBadge } from '@elastic/eui'; +import { isString } from 'lodash/fp'; +import type { CriticalityLevel } from '../../../../../../common/entity_analytics/asset_criticality/types'; +import { CRITICALITY_LEVEL_COLOR } from '../../../../../entity_analytics/components/asset_criticality'; +import { DefaultDraggable } from '../../../../../common/components/draggables'; + +interface Props { + contextId: string; + eventId: string; + fieldName: string; + fieldType: string; + isAggregatable: boolean; + isDraggable: boolean; + value: string | number | undefined | null; +} + +const AssetCriticalityLevelComponent: React.FC<Props> = ({ + contextId, + eventId, + fieldName, + fieldType, + isAggregatable, + isDraggable, + value, +}) => { + const color = isString(value) ? CRITICALITY_LEVEL_COLOR[value as CriticalityLevel] : 'normal'; + + const badge = ( + <EuiBadge color={color} data-test-subj="AssetCriticalityLevel-score-badge"> + {value} + </EuiBadge> + ); + + return isDraggable ? ( + <DefaultDraggable + field={fieldName} + id={`alert-details-value-default-draggable-${contextId}-${eventId}-${fieldName}-${value}`} + fieldType={fieldType} + isAggregatable={isAggregatable} + isDraggable={isDraggable} + value={`${value}`} + tooltipContent={fieldName} + > + {badge} + </DefaultDraggable> + ) : ( + badge + ); +}; + +export const AssetCriticalityLevel = React.memo(AssetCriticalityLevelComponent); +AssetCriticalityLevel.displayName = 'AssetCriticalityLevel'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx index 7062fc7afbb78..040e6335eb8a1 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx @@ -13,6 +13,10 @@ import { isNumber, isEmpty } from 'lodash/fp'; import React from 'react'; import { css } from '@emotion/css'; +import { + ALERT_HOST_CRITICALITY, + ALERT_USER_CRITICALITY, +} from '../../../../../../common/field_maps/field_names'; import { SENTINEL_ONE_AGENT_ID_FIELD } from '../../../../../common/utils/sentinelone_alert_check'; import { SentinelOneAgentStatus } from '../../../../../detections/components/host_isolation/sentinel_one_agent_status'; import { EndpointAgentStatusById } from '../../../../../common/components/endpoint/endpoint_agent_status'; @@ -45,6 +49,7 @@ import { RenderRuleName, renderEventModule, renderUrl } from './formatted_field_ import { RuleStatus } from './rule_status'; import { HostName } from './host_name'; import { UserName } from './user_name'; +import { AssetCriticalityLevel } from './asset_criticality_level'; // simple black-list to prevent dragging and dropping fields such as message name const columnNamesNotDraggable = [MESSAGE_FIELD_NAME]; @@ -256,6 +261,18 @@ const FormattedFieldValueComponent: React.FC<{ iconSide={isButton ? 'right' : undefined} /> ); + } else if (fieldName === ALERT_HOST_CRITICALITY || fieldName === ALERT_USER_CRITICALITY) { + return ( + <AssetCriticalityLevel + contextId={contextId} + eventId={eventId} + fieldName={fieldName} + fieldType={fieldType} + isAggregatable={isAggregatable} + isDraggable={isDraggable} + value={value} + /> + ); } else if (fieldName === AGENT_STATUS_FIELD_NAME) { return ( <EndpointAgentStatusById diff --git a/x-pack/plugins/security_solution/public/timelines/containers/local_storage/index.tsx b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/index.tsx index 8191f800ff934..cc8bafa928180 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/local_storage/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/index.tsx @@ -18,6 +18,7 @@ import { assigneesColumn } from '../../../detections/configurations/security_sol import { ALERTS_TABLE_REGISTRY_CONFIG_IDS, VIEW_SELECTION } from '../../../../common/constants'; import type { DataTablesStorage } from './types'; import { useKibana } from '../../../common/lib/kibana'; +import { migrateEntityRiskLevelColumnTitle } from './migrates_risk_level_title'; export const LOCAL_STORAGE_TABLE_KEY = 'securityDataTable'; const LOCAL_STORAGE_TIMELINE_KEY_LEGACY = 'timelines'; @@ -295,6 +296,7 @@ export const getDataTablesInStorageByIds = (storage: Storage, tableIds: TableIdL migrateAlertTableStateToTriggerActionsState(storage, allDataTables); migrateTriggerActionsVisibleColumnsAlertTable88xTo89(storage); addAssigneesSpecsToSecurityDataTableIfNeeded(storage, allDataTables); + migrateEntityRiskLevelColumnTitle(storage, allDataTables); return tableIds.reduce((acc, tableId) => { const tableModel = allDataTables[tableId]; diff --git a/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.test.tsx b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.test.tsx new file mode 100644 index 0000000000000..a43e0a860e7ac --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.test.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + LOCAL_STORAGE_MIGRATION_KEY, + migrateEntityRiskLevelColumnTitle, +} from './migrates_risk_level_title'; +import type { DataTableState } from '@kbn/securitysolution-data-table'; +import { + hostRiskLevelColumn, + userRiskLevelColumn, +} from '../../../detections/configurations/security_solution_detections/columns'; +import { Storage } from '@kbn/kibana-utils-plugin/public'; +import { localStorageMock } from '../../../common/mock/mock_local_storage'; +import { LOCAL_STORAGE_TABLE_KEY } from '.'; + +const getColumnsBeforeMigration = () => [ + { ...userRiskLevelColumn, displayAsText: undefined }, + { ...hostRiskLevelColumn, displayAsText: undefined }, +]; + +let storage: Storage; + +describe('migrateEntityRiskLevelColumnTitle', () => { + beforeEach(() => { + storage = new Storage(localStorageMock()); + }); + + it('does NOT migrate `columns` when `columns` is not an array', () => { + const dataTableState = { + 'alerts-page': {}, + } as unknown as DataTableState['dataTable']['tableById']; + + migrateEntityRiskLevelColumnTitle(storage, dataTableState); + + expect(dataTableState['alerts-page'].columns).toStrictEqual(undefined); + }); + + it('does not migrates columns if if it has already run once', () => { + storage.set(LOCAL_STORAGE_MIGRATION_KEY, true); + const dataTableState = { + 'alerts-page': { + columns: getColumnsBeforeMigration(), + }, + } as unknown as DataTableState['dataTable']['tableById']; + + migrateEntityRiskLevelColumnTitle(storage, dataTableState); + + expect(dataTableState['alerts-page'].columns).toStrictEqual(getColumnsBeforeMigration()); + }); + + it('migrates columns saved to localStorage', () => { + const dataTableState = { + 'alerts-page': { + columns: getColumnsBeforeMigration(), + }, + } as unknown as DataTableState['dataTable']['tableById']; + + migrateEntityRiskLevelColumnTitle(storage, dataTableState); + + // assert that it mutates the table model + expect(dataTableState['alerts-page'].columns).toStrictEqual([ + userRiskLevelColumn, + hostRiskLevelColumn, + ]); + // assert that it updates the migration flag on storage + expect(storage.get(LOCAL_STORAGE_MIGRATION_KEY)).toEqual(true); + // assert that it updates the table inside local storage + expect(storage.get(LOCAL_STORAGE_TABLE_KEY)['alerts-page'].columns).toStrictEqual([ + userRiskLevelColumn, + hostRiskLevelColumn, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.tsx b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.tsx new file mode 100644 index 0000000000000..8cbae007b3a1f --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.tsx @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DataTableState, TableId } from '@kbn/securitysolution-data-table'; +import { tableEntity, TableEntityType } from '@kbn/securitysolution-data-table'; +import type { Storage } from '@kbn/kibana-utils-plugin/public'; +import { LOCAL_STORAGE_TABLE_KEY } from '.'; +import { + hostRiskLevelColumn, + userRiskLevelColumn, +} from '../../../detections/configurations/security_solution_detections/columns'; + +export const LOCAL_STORAGE_MIGRATION_KEY = + 'securitySolution.dataTable.entityRiskLevelColumnTitleMigration'; + +export const migrateEntityRiskLevelColumnTitle = ( + storage: Storage, + dataTableState: DataTableState['dataTable']['tableById'] +) => { + // Set/Get a flag to prevent migration from running more than once + const hasAlreadyMigrated: boolean = storage.get(LOCAL_STORAGE_MIGRATION_KEY); + if (hasAlreadyMigrated) { + return; + } + storage.set(LOCAL_STORAGE_MIGRATION_KEY, true); + + let updatedTableModel = false; + + for (const [tableId, tableModel] of Object.entries(dataTableState)) { + // Only updates the title for alerts tables + if (tableEntity[tableId as TableId] === TableEntityType.alert) { + // In order to show correct column title after user upgrades to 8.13 we need update the stored table model with the new title. + const columns = tableModel.columns; + if (Array.isArray(columns)) { + columns.forEach((col) => { + if (col.id === userRiskLevelColumn.id) { + col.displayAsText = userRiskLevelColumn.displayAsText; + updatedTableModel = true; + } + + if (col.id === hostRiskLevelColumn.id) { + col.displayAsText = hostRiskLevelColumn.displayAsText; + updatedTableModel = true; + } + }); + } + } + } + if (updatedTableModel) { + storage.set(LOCAL_STORAGE_TABLE_KEY, dataTableState); + } +}; diff --git a/x-pack/plugins/stack_connectors/common/bedrock/constants.ts b/x-pack/plugins/stack_connectors/common/bedrock/constants.ts index 242447d505218..ea3fb7af72fa9 100644 --- a/x-pack/plugins/stack_connectors/common/bedrock/constants.ts +++ b/x-pack/plugins/stack_connectors/common/bedrock/constants.ts @@ -23,6 +23,6 @@ export enum SUB_ACTION { } export const DEFAULT_TOKEN_LIMIT = 8191; -export const DEFAULT_BEDROCK_MODEL = 'anthropic.claude-v2'; +export const DEFAULT_BEDROCK_MODEL = 'anthropic.claude-v2:1'; export const DEFAULT_BEDROCK_URL = `https://bedrock-runtime.us-east-1.amazonaws.com` as const; diff --git a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts index 057780a803560..c26ce8c1e88c3 100644 --- a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts +++ b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts @@ -32,6 +32,8 @@ export const InvokeAIActionParamsSchema = schema.object({ }) ), model: schema.maybe(schema.string()), + temperature: schema.maybe(schema.number()), + stopSequences: schema.maybe(schema.arrayOf(schema.string())), }); export const InvokeAIActionResponseSchema = schema.object({ diff --git a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx index 7678f52321dd3..0ccd8c1d08023 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx @@ -102,7 +102,7 @@ const BedrockParamsFields: React.FunctionComponent<ActionParamsProps<BedrockActi > <EuiFieldText data-test-subj="bedrock-model" - placeholder={'anthropic.claude-v2'} + placeholder={'anthropic.claude-v2:1'} value={model} onChange={(ev) => { editSubActionParams({ model: ev.target.value }); diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts index 3cd2ad2061ffd..3b1cb3bc96ec8 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts @@ -73,7 +73,7 @@ describe('BedrockConnector', () => { 'Content-Type': 'application/json', }, host: 'bedrock-runtime.us-east-1.amazonaws.com', - path: '/model/anthropic.claude-v2/invoke', + path: '/model/anthropic.claude-v2:1/invoke', service: 'bedrock', }, { accessKeyId: '123', secretAccessKey: 'secret' } @@ -137,7 +137,7 @@ describe('BedrockConnector', () => { 'x-amzn-bedrock-accept': '*/*', }, host: 'bedrock-runtime.us-east-1.amazonaws.com', - path: '/model/anthropic.claude-v2/invoke-with-response-stream', + path: '/model/anthropic.claude-v2:1/invoke-with-response-stream', service: 'bedrock', }, { accessKeyId: '123', secretAccessKey: 'secret' } @@ -165,14 +165,53 @@ describe('BedrockConnector', () => { it('formats messages from user, assistant, and system', async () => { await connector.invokeStream({ messages: [ + { + role: 'system', + content: 'Be a good chatbot', + }, { role: 'user', content: 'Hello world', }, + { + role: 'assistant', + content: 'Hi, I am a good chatbot', + }, + { + role: 'user', + content: 'What is 2+2?', + }, + ], + }); + expect(mockRequest).toHaveBeenCalledWith({ + signed: true, + responseType: 'stream', + url: `${DEFAULT_BEDROCK_URL}/model/${DEFAULT_BEDROCK_MODEL}/invoke-with-response-stream`, + method: 'post', + responseSchema: StreamingResponseSchema, + data: JSON.stringify({ + prompt: + 'Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:', + max_tokens_to_sample: DEFAULT_TOKEN_LIMIT, + temperature: 0.5, + stop_sequences: ['\n\nHuman:'], + }), + }); + }); + + it('formats the system message as a user message for claude<2.1', async () => { + const modelOverride = 'anthropic.claude-v2'; + + await connector.invokeStream({ + messages: [ { role: 'system', content: 'Be a good chatbot', }, + { + role: 'user', + content: 'Hello world', + }, { role: 'assistant', content: 'Hi, I am a good chatbot', @@ -182,16 +221,17 @@ describe('BedrockConnector', () => { content: 'What is 2+2?', }, ], + model: modelOverride, }); expect(mockRequest).toHaveBeenCalledWith({ signed: true, responseType: 'stream', - url: `${DEFAULT_BEDROCK_URL}/model/${DEFAULT_BEDROCK_MODEL}/invoke-with-response-stream`, + url: `${DEFAULT_BEDROCK_URL}/model/${modelOverride}/invoke-with-response-stream`, method: 'post', responseSchema: StreamingResponseSchema, data: JSON.stringify({ prompt: - '\n\nHuman:Hello world\n\nHuman:Be a good chatbot\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:', + '\n\nHuman:Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:', max_tokens_to_sample: DEFAULT_TOKEN_LIMIT, temperature: 0.5, stop_sequences: ['\n\nHuman:'], @@ -244,14 +284,14 @@ describe('BedrockConnector', () => { it('formats messages from user, assistant, and system', async () => { const response = await connector.invokeAI({ messages: [ - { - role: 'user', - content: 'Hello world', - }, { role: 'system', content: 'Be a good chatbot', }, + { + role: 'user', + content: 'Hello world', + }, { role: 'assistant', content: 'Hi, I am a good chatbot', @@ -271,7 +311,7 @@ describe('BedrockConnector', () => { responseSchema: RunActionResponseSchema, data: JSON.stringify({ prompt: - '\n\nHuman:Hello world\n\nHuman:Be a good chatbot\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:', + 'Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:', max_tokens_to_sample: DEFAULT_TOKEN_LIMIT, temperature: 0.5, stop_sequences: ['\n\nHuman:'], diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts index f70a592509776..3fdbaae1d702a 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts @@ -26,7 +26,11 @@ import type { InvokeAIActionResponse, StreamActionParams, } from '../../../common/bedrock/types'; -import { SUB_ACTION, DEFAULT_TOKEN_LIMIT } from '../../../common/bedrock/constants'; +import { + SUB_ACTION, + DEFAULT_TOKEN_LIMIT, + DEFAULT_BEDROCK_MODEL, +} from '../../../common/bedrock/constants'; import { DashboardActionParams, DashboardActionResponse, @@ -233,9 +237,14 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B * @param messages An array of messages to be sent to the API * @param model Optional model to be used for the API request. If not provided, the default model from the connector will be used. */ - public async invokeStream({ messages, model }: InvokeAIActionParams): Promise<IncomingMessage> { + public async invokeStream({ + messages, + model, + stopSequences, + temperature, + }: InvokeAIActionParams): Promise<IncomingMessage> { const res = (await this.streamApi({ - body: JSON.stringify(formatBedrockBody({ messages })), + body: JSON.stringify(formatBedrockBody({ messages, model, stopSequences, temperature })), model, })) as unknown as IncomingMessage; return res; @@ -250,20 +259,43 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B messages, model, }: InvokeAIActionParams): Promise<InvokeAIActionResponse> { - const res = await this.runApi({ body: JSON.stringify(formatBedrockBody({ messages })), model }); + const res = await this.runApi({ + body: JSON.stringify(formatBedrockBody({ messages, model })), + model, + }); return { message: res.completion.trim() }; } } const formatBedrockBody = ({ + model = DEFAULT_BEDROCK_MODEL, messages, + stopSequences = ['\n\nHuman:'], + temperature = 0.5, }: { + model?: string; messages: Array<{ role: string; content: string }>; + stopSequences?: string[]; + temperature?: number; }) => { const combinedMessages = messages.reduce((acc: string, message) => { const { role, content } = message; - // Bedrock only has Assistant and Human, so 'system' and 'user' will be converted to Human - const bedrockRole = role === 'assistant' ? '\n\nAssistant:' : '\n\nHuman:'; + const [, , modelName, majorVersion, minorVersion] = + (model || '').match(/(\w+)\.(.*)-v(\d+)(?::(\d+))?/) || []; + // Claude only has Assistant and Human, so 'user' will be converted to Human + let bedrockRole: string; + + if ( + role === 'system' && + modelName === 'claude' && + Number(majorVersion) >= 2 && + Number(minorVersion) >= 1 + ) { + bedrockRole = ''; + } else { + bedrockRole = role === 'assistant' ? '\n\nAssistant:' : '\n\nHuman:'; + } + return `${acc}${bedrockRole}${content}`; }, ''); @@ -271,8 +303,8 @@ const formatBedrockBody = ({ // end prompt in "Assistant:" to avoid the model starting its message with "Assistant:" prompt: `${combinedMessages} \n\nAssistant:`, max_tokens_to_sample: DEFAULT_TOKEN_LIMIT, - temperature: 0.5, + temperature, // prevent model from talking to itself - stop_sequences: ['\n\nHuman:'], + stop_sequences: stopSequences, }; }; diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts index 5f295b8c39367..688148d51ed63 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts @@ -10,7 +10,10 @@ import { SubActionConnectorType, ValidatorType, } from '@kbn/actions-plugin/server/sub_action_framework/types'; -import { GenerativeAIForSecurityConnectorFeatureId } from '@kbn/actions-plugin/common'; +import { + GenerativeAIForObservabilityConnectorFeatureId, + GenerativeAIForSecurityConnectorFeatureId, +} from '@kbn/actions-plugin/common'; import { urlAllowListValidator } from '@kbn/actions-plugin/server'; import { ValidatorServices } from '@kbn/actions-plugin/server/types'; import { assertURL } from '@kbn/actions-plugin/server/sub_action_framework/helpers/validators'; @@ -29,7 +32,10 @@ export const getConnectorType = (): SubActionConnectorType<Config, Secrets> => ( secrets: SecretsSchema, }, validators: [{ type: ValidatorType.CONFIG, validator: configValidator }], - supportedFeatureIds: [GenerativeAIForSecurityConnectorFeatureId], + supportedFeatureIds: [ + GenerativeAIForSecurityConnectorFeatureId, + GenerativeAIForObservabilityConnectorFeatureId, + ], minimumLicenseRequired: 'enterprise' as const, renderParameterTemplates, }); diff --git a/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts b/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts index 716e1f8dcb83f..c7ee109d17b11 100644 --- a/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts +++ b/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts @@ -308,6 +308,7 @@ describe('task state validation', () => { it('should fail the task run when setting allow_reading_invalid_state:false and reading an invalid state', async () => { const logSpy = jest.spyOn(pollingLifecycleOpts.logger, 'warn'); + const updateSpy = jest.spyOn(pollingLifecycleOpts.taskStore, 'bulkUpdate'); const id = uuidV4(); await injectTask(kibanaServer.coreStart.elasticsearch.client.asInternalUser, { @@ -331,8 +332,9 @@ describe('task state validation', () => { expect(logSpy.mock.calls[0][0]).toBe( `Task (fooType/${id}) has a validation error: [foo]: expected value of type [string] but got [boolean]` ); - expect(logSpy.mock.calls[1][0]).toBe( - `Task fooType \"${id}\" failed in attempt to run: [foo]: expected value of type [string] but got [boolean]` + expect(updateSpy).toHaveBeenCalledWith( + expect.arrayContaining([expect.objectContaining({ id, taskType: 'fooType' })]), + { validate: false } ); }); }); diff --git a/x-pack/plugins/task_manager/server/task.ts b/x-pack/plugins/task_manager/server/task.ts index c71f8b42185ca..0d064153859a5 100644 --- a/x-pack/plugins/task_manager/server/task.ts +++ b/x-pack/plugins/task_manager/server/task.ts @@ -50,6 +50,7 @@ export type SuccessfulRunResult = { state: Record<string, unknown>; taskRunError?: DecoratedError; skipAttempts?: number; + shouldValidate?: boolean; } & ( | // ensure a SuccessfulRunResult can either specify a new `runAt` or a new `schedule`, but not both { diff --git a/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts b/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts index 8a96405abfed6..6735b3c0602b8 100644 --- a/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts +++ b/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts @@ -1082,6 +1082,7 @@ describe('TaskManagerRunner', () => { await runner.run(); expect(store.update).toHaveBeenCalledTimes(1); + expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true }); const instance = store.update.mock.calls[0][0]; expect(instance.runAt.getTime()).toEqual(nextRetry.getTime()); @@ -1113,6 +1114,8 @@ describe('TaskManagerRunner', () => { await runner.run(); expect(store.update).toHaveBeenCalledTimes(1); + expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true }); + const instance = store.update.mock.calls[0][0]; const minRunAt = Date.now(); @@ -1179,6 +1182,8 @@ describe('TaskManagerRunner', () => { await runner.run(); expect(store.update).toHaveBeenCalledTimes(1); + expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true }); + sinon.assert.notCalled(getRetryStub); const instance = store.update.mock.calls[0][0]; @@ -1252,6 +1257,7 @@ describe('TaskManagerRunner', () => { new Date(Date.now() + intervalSeconds * 1000).getTime() ); expect(instance.enabled).not.toBeDefined(); + expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true }); }); test('throws error when the task has invalid state', async () => { @@ -1266,7 +1272,7 @@ describe('TaskManagerRunner', () => { stateVersion: 4, }; - const { runner, logger } = await readyToRunStageSetup({ + const { runner, logger, store } = await readyToRunStageSetup({ instance: mockTaskInstance, definitions: { bar: { @@ -1308,13 +1314,19 @@ describe('TaskManagerRunner', () => { }, }); - expect(() => runner.run()).rejects.toMatchInlineSnapshot( - `[Error: [foo]: expected value of type [string] but got [boolean]]` - ); + expect(await runner.run()).toEqual({ + error: { + error: new Error('[foo]: expected value of type [string] but got [boolean]'), + shouldValidate: false, + state: { bar: 'test', baz: 'test', foo: true }, + }, + tag: 'err', + }); expect(logger.warn).toHaveBeenCalledTimes(1); expect(logger.warn).toHaveBeenCalledWith( 'Task (bar/foo) has a validation error: [foo]: expected value of type [string] but got [boolean]' ); + expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: false }); }); test('does not throw error and runs when the task has invalid state and allowReadingInvalidState = true', async () => { diff --git a/x-pack/plugins/task_manager/server/task_running/task_runner.ts b/x-pack/plugins/task_manager/server/task_running/task_runner.ts index ab86d83e99310..faea2bfb7e446 100644 --- a/x-pack/plugins/task_manager/server/task_running/task_runner.ts +++ b/x-pack/plugins/task_manager/server/task_running/task_runner.ts @@ -314,16 +314,30 @@ export class TaskManagerRunner implements TaskRunner { const apmTrans = apm.startTransaction(this.taskType, TASK_MANAGER_RUN_TRANSACTION_TYPE, { childOf: this.instance.task.traceparent, }); + const stopTaskTimer = startTaskTimerWithEventLoopMonitoring(this.eventLoopDelayConfig); // Validate state - const validatedTaskInstance = this.validateTaskState(this.instance.task); + const stateValidationResult = this.validateTaskState(this.instance.task); + + if (stateValidationResult.error) { + const processedResult = await withSpan({ name: 'process result', type: 'task manager' }, () => + this.processResult( + asErr({ + error: stateValidationResult.error, + state: stateValidationResult.taskInstance.state, + shouldValidate: false, + }), + stopTaskTimer() + ) + ); + if (apmTrans) apmTrans.end('failure'); + return processedResult; + } const modifiedContext = await this.beforeRun({ - taskInstance: validatedTaskInstance, + taskInstance: stateValidationResult.taskInstance, }); - const stopTaskTimer = startTaskTimerWithEventLoopMonitoring(this.eventLoopDelayConfig); - this.onTaskEvent( asTaskManagerStatEvent( 'runDelay', @@ -411,11 +425,12 @@ export class TaskManagerRunner implements TaskRunner { private validateTaskState(taskInstance: ConcreteTaskInstance) { const { taskType, id } = taskInstance; try { - const validatedTask = this.taskValidator.getValidatedTaskInstanceFromReading(taskInstance); - return validatedTask; + const validatedTaskInstance = + this.taskValidator.getValidatedTaskInstanceFromReading(taskInstance); + return { taskInstance: validatedTaskInstance, error: null }; } catch (error) { this.logger.warn(`Task (${taskType}/${id}) has a validation error: ${error.message}`); - throw error; + return { taskInstance, error }; } } @@ -723,6 +738,7 @@ export class TaskManagerRunner implements TaskRunner { this.instance = asRan(this.instance.task); await this.removeTask(); } else { + const { shouldValidate = true } = unwrap(result); this.instance = asRan( await this.bufferedTaskStore.update( defaults( @@ -735,7 +751,7 @@ export class TaskManagerRunner implements TaskRunner { }, taskWithoutEnabled(this.instance.task) ), - { validate: true } + { validate: shouldValidate } ) ); } diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx index 2031c9a1f3fe2..2619ef2b25258 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx @@ -37,9 +37,9 @@ export const AlertSummaryWidget = ({ useEffect(() => { if (!isLoading && onLoaded) { - onLoaded(); + onLoaded({ activeAlertCount, recoveredAlertCount }); } - }, [isLoading, onLoaded]); + }, [activeAlertCount, isLoading, onLoaded, recoveredAlertCount]); if (isLoading) return <AlertSummaryWidgetLoader fullSize={fullSize} isLoadingWithoutChart={hideChart} />; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts index 48a49acf5ad7c..1bb02adff0653 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts @@ -30,6 +30,11 @@ export interface ChartProps { onBrushEnd?: BrushEndListener; } +interface AlertsCount { + activeAlertCount: number; + recoveredAlertCount: number; +} + export interface AlertSummaryWidgetProps { featureIds?: ValidFeatureId[]; filter?: estypes.QueryDslQueryContainer; @@ -38,5 +43,5 @@ export interface AlertSummaryWidgetProps { timeRange: AlertSummaryTimeRange; chartProps: ChartProps; hideChart?: boolean; - onLoaded?: () => void; + onLoaded?: (alertsCount?: AlertsCount) => void; } diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/index.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/index.tsx index dfab594febf10..1a99e346ed808 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/index.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/index.tsx @@ -7,6 +7,13 @@ import { lazy } from 'react'; import { suspendedComponentWithProps } from '../../lib/suspended_component_with_props'; +import type { RuleAddComponent } from './rule_add'; +import type { RuleEditComponent } from './rule_edit'; -export const RuleAdd = suspendedComponentWithProps(lazy(() => import('./rule_add'))); -export const RuleEdit = suspendedComponentWithProps(lazy(() => import('./rule_edit'))); +export const RuleAdd = suspendedComponentWithProps( + lazy(() => import('./rule_add')) +) as RuleAddComponent; // `React.lazy` is not typed correctly to support generics so casting back to imported component + +export const RuleEdit = suspendedComponentWithProps( + lazy(() => import('./rule_edit')) +) as RuleEditComponent; // `React.lazy` is not typed correctly to support generics so casting back to imported component diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_add.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_add.tsx index 07264709dd544..19eb8da4bf0d3 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_add.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_add.tsx @@ -15,6 +15,7 @@ import { parseRuleCircuitBreakerErrorMessage } from '@kbn/alerting-plugin/common import { Rule, RuleTypeParams, + RuleTypeMetaData, RuleUpdates, RuleFlyoutCloseReason, IErrorObject, @@ -49,7 +50,12 @@ const defaultCreateRuleErrorMessage = i18n.translate( } ); -const RuleAdd = ({ +export type RuleAddComponent = typeof RuleAdd; + +const RuleAdd = < + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData +>({ consumer, ruleTypeRegistry, actionTypeRegistry, @@ -67,7 +73,7 @@ const RuleAdd = ({ useRuleProducer, initialSelectedConsumer, ...props -}: RuleAddProps) => { +}: RuleAddProps<Params, MetaData>) => { const onSaveHandler = onSave ?? reloadRules; const [metadata, setMetadata] = useState(initialMetadata); const onChangeMetaData = useCallback((newMetadata) => setMetadata(newMetadata), []); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_edit.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_edit.tsx index 0aebaaaa29882..975881e516e45 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_edit.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_edit.tsx @@ -34,6 +34,8 @@ import { RuleEditProps, IErrorObject, RuleType, + RuleTypeParams, + RuleTypeMetaData, TriggersActionsUiConfig, RuleNotifyWhenType, } from '../../../types'; @@ -81,7 +83,12 @@ const cloneAndMigrateRule = (initialRule: Rule) => { return clonedRule; }; -export const RuleEdit = ({ +export type RuleEditComponent = typeof RuleEdit; + +export const RuleEdit = < + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData +>({ initialRule, onClose, reloadRules, @@ -91,7 +98,7 @@ export const RuleEdit = ({ actionTypeRegistry, metadata: initialMetadata, ...props -}: RuleEditProps) => { +}: RuleEditProps<Params, MetaData>) => { const onSaveHandler = onSave ?? reloadRules; const [{ rule }, dispatch] = useReducer(ruleReducer as ConcreteRuleReducer, { rule: cloneAndMigrateRule(initialRule), diff --git a/x-pack/plugins/triggers_actions_ui/public/common/get_add_rule_flyout.tsx b/x-pack/plugins/triggers_actions_ui/public/common/get_add_rule_flyout.tsx index 23f751201d1be..c6a79b4c6e82d 100644 --- a/x-pack/plugins/triggers_actions_ui/public/common/get_add_rule_flyout.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/common/get_add_rule_flyout.tsx @@ -9,11 +9,14 @@ import React from 'react'; import { QueryClientProvider } from '@tanstack/react-query'; import { ConnectorProvider } from '../application/context/connector_context'; import { RuleAdd } from '../application/sections/rule_form'; -import type { ConnectorServices, RuleAddProps } from '../types'; +import type { ConnectorServices, RuleAddProps, RuleTypeParams, RuleTypeMetaData } from '../types'; import { queryClient } from '../application/query_client'; -export const getAddRuleFlyoutLazy = ( - props: RuleAddProps & { connectorServices: ConnectorServices } +export const getAddRuleFlyoutLazy = < + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData +>( + props: RuleAddProps<Params, MetaData> & { connectorServices: ConnectorServices } ) => { return ( <ConnectorProvider value={{ services: props.connectorServices }}> diff --git a/x-pack/plugins/triggers_actions_ui/public/common/get_edit_rule_flyout.tsx b/x-pack/plugins/triggers_actions_ui/public/common/get_edit_rule_flyout.tsx index 2d99e3911a168..f3fbccce267c5 100644 --- a/x-pack/plugins/triggers_actions_ui/public/common/get_edit_rule_flyout.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/common/get_edit_rule_flyout.tsx @@ -9,11 +9,14 @@ import React from 'react'; import { QueryClientProvider } from '@tanstack/react-query'; import { ConnectorProvider } from '../application/context/connector_context'; import { RuleEdit } from '../application/sections/rule_form'; -import type { ConnectorServices, RuleEditProps as AlertEditProps } from '../types'; +import type { ConnectorServices, RuleEditProps, RuleTypeParams, RuleTypeMetaData } from '../types'; import { queryClient } from '../application/query_client'; -export const getEditRuleFlyoutLazy = ( - props: AlertEditProps & { connectorServices: ConnectorServices } +export const getEditRuleFlyoutLazy = < + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData +>( + props: RuleEditProps<Params, MetaData> & { connectorServices: ConnectorServices } ) => { return ( <ConnectorProvider value={{ services: props.connectorServices }}> diff --git a/x-pack/plugins/triggers_actions_ui/public/mocks.ts b/x-pack/plugins/triggers_actions_ui/public/mocks.ts index 223a54205cb48..48691c15ed62f 100644 --- a/x-pack/plugins/triggers_actions_ui/public/mocks.ts +++ b/x-pack/plugins/triggers_actions_ui/public/mocks.ts @@ -15,8 +15,6 @@ import { getEditRuleFlyoutLazy } from './common/get_edit_rule_flyout'; import { TypeRegistry } from './application/type_registry'; import { ActionTypeModel, - RuleAddProps, - RuleEditProps, RuleTypeModel, AlertsTableProps, FieldBrowserProps, @@ -73,7 +71,7 @@ function createStartMock(): TriggersAndActionsUIPublicPluginStart { connectorServices, }); }, - getAddRuleFlyout: (props: Omit<RuleAddProps, 'actionTypeRegistry' | 'ruleTypeRegistry'>) => { + getAddRuleFlyout: (props) => { return getAddRuleFlyoutLazy({ ...props, actionTypeRegistry, @@ -81,7 +79,7 @@ function createStartMock(): TriggersAndActionsUIPublicPluginStart { connectorServices, }); }, - getEditRuleFlyout: (props: Omit<RuleEditProps, 'actionTypeRegistry' | 'ruleTypeRegistry'>) => { + getEditRuleFlyout: (props) => { return getEditRuleFlyoutLazy({ ...props, actionTypeRegistry, diff --git a/x-pack/plugins/triggers_actions_ui/public/plugin.ts b/x-pack/plugins/triggers_actions_ui/public/plugin.ts index 872528a9a5f85..bcd639e21a2ff 100644 --- a/x-pack/plugins/triggers_actions_ui/public/plugin.ts +++ b/x-pack/plugins/triggers_actions_ui/public/plugin.ts @@ -62,6 +62,8 @@ import type { RuleAddProps, RuleEditProps, RuleTypeModel, + RuleTypeParams, + RuleTypeMetaData, AlertsTableProps, RuleStatusDropdownProps, RuleTagFilterProps, @@ -115,12 +117,18 @@ export interface TriggersAndActionsUIPublicPluginStart { getEditConnectorFlyout: ( props: Omit<EditConnectorFlyoutProps, 'actionTypeRegistry'> ) => ReactElement<EditConnectorFlyoutProps>; - getAddRuleFlyout: ( - props: Omit<RuleAddProps, 'actionTypeRegistry' | 'ruleTypeRegistry'> - ) => ReactElement<RuleAddProps>; - getEditRuleFlyout: ( - props: Omit<RuleEditProps, 'actionTypeRegistry' | 'ruleTypeRegistry'> - ) => ReactElement<RuleEditProps>; + getAddRuleFlyout: < + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData + >( + props: Omit<RuleAddProps<Params, MetaData>, 'actionTypeRegistry' | 'ruleTypeRegistry'> + ) => ReactElement<RuleAddProps<Params, MetaData>>; + getEditRuleFlyout: < + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData + >( + props: Omit<RuleEditProps<Params, MetaData>, 'actionTypeRegistry' | 'ruleTypeRegistry'> + ) => ReactElement<RuleEditProps<Params, MetaData>>; getAlertsTable: (props: AlertsTableProps) => ReactElement<AlertsTableProps>; getAlertsTableDefaultAlertActions: <P extends AlertActionsProps>( props: P @@ -403,7 +411,7 @@ export class Plugin connectorServices: this.connectorServices!, }); }, - getAddRuleFlyout: (props: Omit<RuleAddProps, 'actionTypeRegistry' | 'ruleTypeRegistry'>) => { + getAddRuleFlyout: (props) => { return getAddRuleFlyoutLazy({ ...props, actionTypeRegistry: this.actionTypeRegistry, @@ -411,9 +419,7 @@ export class Plugin connectorServices: this.connectorServices!, }); }, - getEditRuleFlyout: ( - props: Omit<RuleEditProps, 'actionTypeRegistry' | 'ruleTypeRegistry'> - ) => { + getEditRuleFlyout: (props) => { return getEditRuleFlyoutLazy({ ...props, actionTypeRegistry: this.actionTypeRegistry, diff --git a/x-pack/plugins/triggers_actions_ui/public/types.ts b/x-pack/plugins/triggers_actions_ui/public/types.ts index b47d80a0839e5..36cc294bbda5f 100644 --- a/x-pack/plugins/triggers_actions_ui/public/types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/types.ts @@ -51,6 +51,7 @@ import { AlertingFrameworkHealth, RuleNotifyWhenType, RuleTypeParams, + RuleTypeMetaData, ActionVariable, RuleLastRun, MaintenanceWindow, @@ -127,6 +128,7 @@ export type { AlertingFrameworkHealth, RuleNotifyWhenType, RuleTypeParams, + RuleTypeMetaData, ResolvedRule, SanitizedRule, RuleStatusDropdownProps, @@ -412,8 +414,11 @@ export enum EditConnectorTabs { Test = 'test', } -export interface RuleEditProps<MetaData = Record<string, any>> { - initialRule: Rule; +export interface RuleEditProps< + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData +> { + initialRule: Rule<Params>; ruleTypeRegistry: RuleTypeRegistryContract; actionTypeRegistry: ActionTypeRegistryContract; onClose: (reason: RuleFlyoutCloseReason, metadata?: MetaData) => void; @@ -425,14 +430,27 @@ export interface RuleEditProps<MetaData = Record<string, any>> { ruleType?: RuleType<string, string>; } -export interface RuleAddProps<MetaData = Record<string, any>> { +export interface RuleAddProps< + Params extends RuleTypeParams = RuleTypeParams, + MetaData extends RuleTypeMetaData = RuleTypeMetaData +> { + /** + * ID of the feature this rule should be created for. + * + * Notes: + * - The feature needs to be registered using `featuresPluginSetup.registerKibanaFeature()` API during your plugin's setup phase. + * - The user needs to have permission to access the feature in order to create the rule. + * */ consumer: string; ruleTypeRegistry: RuleTypeRegistryContract; actionTypeRegistry: ActionTypeRegistryContract; onClose: (reason: RuleFlyoutCloseReason, metadata?: MetaData) => void; ruleTypeId?: string; + /** + * Determines whether the user should be able to change the rule type in the UI. + */ canChangeTrigger?: boolean; - initialValues?: Partial<Rule>; + initialValues?: Partial<Rule<Params>>; /** @deprecated use `onSave` as a callback after an alert is saved*/ reloadRules?: () => Promise<void>; hideGrouping?: boolean; @@ -445,8 +463,8 @@ export interface RuleAddProps<MetaData = Record<string, any>> { useRuleProducer?: boolean; initialSelectedConsumer?: RuleCreationValidConsumer | null; } -export interface RuleDefinitionProps { - rule: Rule; +export interface RuleDefinitionProps<Params extends RuleTypeParams = RuleTypeParams> { + rule: Rule<Params>; ruleTypeRegistry: RuleTypeRegistryContract; actionTypeRegistry: ActionTypeRegistryContract; onEditRule: () => Promise<void>; diff --git a/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts b/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts index 29e77feb5edaf..18051754cc77a 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts @@ -29,7 +29,7 @@ export class BedrockSimulator extends Simulator { return BedrockSimulator.sendErrorResponse(response); } - if (request.url === '/model/anthropic.claude-v2/invoke-with-response-stream') { + if (request.url === '/model/anthropic.claude-v2:1/invoke-with-response-stream') { return BedrockSimulator.sendStreamResponse(response); } diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts index 14ec27598a60f..fc0ca3378d8c0 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts @@ -27,7 +27,7 @@ const secrets = { }; const defaultConfig = { - defaultModel: 'anthropic.claude-v2', + defaultModel: 'anthropic.claude-v2:1', }; // eslint-disable-next-line import/no-default-export @@ -380,14 +380,14 @@ export default function bedrockTest({ getService }: FtrProviderContext) { subAction: 'invokeAI', subActionParams: { messages: [ - { - role: 'user', - content: 'Hello world', - }, { role: 'system', content: 'Be a good chatbot', }, + { + role: 'user', + content: 'Hello world', + }, { role: 'assistant', content: 'Hi, I am a good chatbot', @@ -404,7 +404,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { expect(simulator.requestData).to.eql({ prompt: - '\n\nHuman:Hello world\n\nHuman:Be a good chatbot\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:', + 'Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:', max_tokens_to_sample: DEFAULT_TOKEN_LIMIT, temperature: 0.5, stop_sequences: ['\n\nHuman:'], diff --git a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts index 6e8fbffbe0416..e929cbff2f188 100644 --- a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts +++ b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts @@ -36,9 +36,9 @@ export function templatesApi(getService: FtrProviderContext['getService']) { .send(payload); // Delete all templates created during tests - const cleanUpTemplates = async () => { + const cleanUpTemplates = async (additionalRequestHeaders: object = {}) => { try { - await deleteTemplates(templatesCreated); + await deleteTemplates(templatesCreated).set(additionalRequestHeaders); templatesCreated = []; } catch (e) { // Silently swallow errors diff --git a/x-pack/test/api_integration/apis/management/index_management/templates.ts b/x-pack/test/api_integration/apis/management/index_management/templates.ts index a3b2ce50e04b5..6d0e79993d040 100644 --- a/x-pack/test/api_integration/apis/management/index_management/templates.ts +++ b/x-pack/test/api_integration/apis/management/index_management/templates.ts @@ -96,6 +96,7 @@ export default function ({ getService }: FtrProviderContext) { 'hasMappings', 'priority', 'composedOf', + 'ignoreMissingComponentTemplates', 'version', '_kbnMeta', ].sort(); @@ -119,6 +120,7 @@ export default function ({ getService }: FtrProviderContext) { 'version', '_kbnMeta', 'composedOf', + 'ignoreMissingComponentTemplates', ].sort(); expect(Object.keys(legacyTemplateFound).sort()).to.eql(expectedLegacyKeys); @@ -139,6 +141,7 @@ export default function ({ getService }: FtrProviderContext) { 'hasMappings', 'priority', 'composedOf', + 'ignoreMissingComponentTemplates', 'dataStream', 'version', '_kbnMeta', @@ -162,6 +165,7 @@ export default function ({ getService }: FtrProviderContext) { 'hasMappings', 'priority', 'composedOf', + 'ignoreMissingComponentTemplates', 'version', '_kbnMeta', ].sort(); @@ -183,6 +187,7 @@ export default function ({ getService }: FtrProviderContext) { 'indexPatterns', 'template', 'composedOf', + 'ignoreMissingComponentTemplates', 'priority', 'version', '_kbnMeta', @@ -207,6 +212,7 @@ export default function ({ getService }: FtrProviderContext) { 'version', '_kbnMeta', 'composedOf', + 'ignoreMissingComponentTemplates', ].sort(); const expectedTemplateKeys = ['aliases', 'mappings', 'settings'].sort(); diff --git a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts index 06a67a13e425c..a257ff97933d9 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts @@ -41,43 +41,46 @@ export default function (providerContext: FtrProviderContext) { skipIfNoDockerRegistry(providerContext); setupFleetAndAgents(providerContext); + const writeMetricsDoc = (namespace: string) => + es.transport.request( + { + method: 'POST', + path: `/${metricsTemplateName}-${namespace}/_doc?refresh=true`, + body: { + '@timestamp': new Date().toISOString(), + logs_test_name: 'test', + data_stream: { + dataset: `${pkgName}.test_metrics`, + namespace, + type: 'metrics', + }, + }, + }, + { meta: true } + ); + + const writeLogsDoc = (namespace: string) => + es.transport.request( + { + method: 'POST', + path: `/${logsTemplateName}-${namespace}/_doc?refresh=true`, + body: { + '@timestamp': new Date().toISOString(), + logs_test_name: 'test', + data_stream: { + dataset: `${pkgName}.test_logs`, + namespace, + type: 'logs', + }, + }, + }, + { meta: true } + ); beforeEach(async () => { await installPackage(pkgName, pkgVersion); await Promise.all( namespaces.map(async (namespace) => { - const createLogsRequest = es.transport.request( - { - method: 'POST', - path: `/${logsTemplateName}-${namespace}/_doc`, - body: { - '@timestamp': '2015-01-01', - logs_test_name: 'test', - data_stream: { - dataset: `${pkgName}.test_logs`, - namespace, - type: 'logs', - }, - }, - }, - { meta: true } - ); - const createMetricsRequest = es.transport.request( - { - method: 'POST', - path: `/${metricsTemplateName}-${namespace}/_doc`, - body: { - '@timestamp': '2015-01-01', - logs_test_name: 'test', - data_stream: { - dataset: `${pkgName}.test_metrics`, - namespace, - type: 'metrics', - }, - }, - }, - { meta: true } - ); - return Promise.all([createLogsRequest, createMetricsRequest]); + return Promise.all([writeLogsDoc(namespace), writeMetricsDoc(namespace)]); }) ); }); @@ -141,7 +144,11 @@ export default function (providerContext: FtrProviderContext) { it('after update, it should have rolled over logs datastream because mappings are not compatible and not metrics', async function () { await installPackage(pkgName, pkgUpdateVersion); + await asyncForEach(namespaces, async (namespace) => { + // write doc as rollover is lazy + await writeLogsDoc(namespace); + await writeMetricsDoc(namespace); const resLogsDatastream = await es.transport.request<any>( { method: 'GET', @@ -266,6 +273,8 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); + // Write a doc to trigger lazy rollover + await writeLogsDoc('default'); // Datastream should have been rolled over expect(await getLogsDefaultBackingIndicesLength()).to.be(2); }); @@ -303,26 +312,29 @@ export default function (providerContext: FtrProviderContext) { skipIfNoDockerRegistry(providerContext); setupFleetAndAgents(providerContext); - beforeEach(async () => { - await installPackage(pkgName, pkgVersion); - - // Create a sample document so the data stream is created - await es.transport.request( + const writeMetricDoc = (body: any = {}) => + es.transport.request( { method: 'POST', - path: `/${metricsTemplateName}-${namespace}/_doc`, + path: `/${metricsTemplateName}-${namespace}/_doc?refresh=true`, body: { - '@timestamp': '2015-01-01', + '@timestamp': new Date().toISOString(), logs_test_name: 'test', data_stream: { dataset: `${pkgName}.test_logs`, namespace, type: 'logs', }, + ...body, }, }, { meta: true } ); + beforeEach(async () => { + await installPackage(pkgName, pkgVersion); + + // Create a sample document so the data stream is created + await writeMetricDoc(); }); afterEach(async () => { @@ -340,6 +352,10 @@ export default function (providerContext: FtrProviderContext) { it('rolls over data stream when index_mode: time_series is set in the updated package version', async () => { await installPackage(pkgName, pkgUpdateVersion); + // Write a doc so lazy rollover can happen + await writeMetricDoc({ + some_field: 'test', + }); const resMetricsDatastream = await es.transport.request<any>( { method: 'GET', diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts index 2fe976352944a..2ec6fb92000e3 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts @@ -34,46 +34,50 @@ export default function (providerContext: FtrProviderContext) { .send({ force: true }) .expect(200); - await es.index({ - index: 'metrics-apm.service_summary.10m-default', - document: { - '@timestamp': '2023-05-30T07:50:00.000Z', - agent: { - name: 'go', - }, - data_stream: { - dataset: 'apm.service_summary.10m', - namespace: 'default', - type: 'metrics', - }, - ecs: { - version: '8.6.0-dev', - }, - event: { - agent_id_status: 'missing', - ingested: '2023-05-30T07:57:12Z', - }, - metricset: { - interval: '10m', - name: 'service_summary', - }, - observer: { - hostname: '047e282994fb', - type: 'apm-server', - version: '8.7.0', - }, - processor: { - event: 'metric', - name: 'metric', - }, - service: { - language: { + const writeDoc = () => + es.index({ + refresh: true, + index: 'metrics-apm.service_summary.10m-default', + document: { + '@timestamp': '2023-05-30T07:50:00.000Z', + agent: { name: 'go', }, - name: '___main_elastic_cloud_87_ilm_fix', + data_stream: { + dataset: 'apm.service_summary.10m', + namespace: 'default', + type: 'metrics', + }, + ecs: { + version: '8.6.0-dev', + }, + event: { + agent_id_status: 'missing', + ingested: '2023-05-30T07:57:12Z', + }, + metricset: { + interval: '10m', + name: 'service_summary', + }, + observer: { + hostname: '047e282994fb', + type: 'apm-server', + version: '8.7.0', + }, + processor: { + event: 'metric', + name: 'metric', + }, + service: { + language: { + name: 'go', + }, + name: '___main_elastic_cloud_87_ilm_fix', + }, }, - }, - }); + }); + + await writeDoc(); await supertest .post(`/api/fleet/epm/packages/apm/8.8.0`) @@ -81,6 +85,8 @@ export default function (providerContext: FtrProviderContext) { .send({ force: true }) .expect(200); + // Rollover are lazy need to write a new doc + await writeDoc(); const ds = await es.indices.get({ index: 'metrics-apm.service_summary*', expand_wildcards: ['open', 'hidden'], diff --git a/x-pack/test/functional/apps/infra/node_details.ts b/x-pack/test/functional/apps/infra/node_details.ts index 172ac410eb427..42147652f34a5 100644 --- a/x-pack/test/functional/apps/infra/node_details.ts +++ b/x-pack/test/functional/apps/infra/node_details.ts @@ -204,6 +204,15 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await pageObjects.assetDetails.overviewAlertsTitleExists(); }); + it('should show / hide alerts section with no alerts and show / hide closed section content', async () => { + await pageObjects.assetDetails.alertsSectionCollapsibleExist(); + // Collapsed by default + await pageObjects.assetDetails.alertsSectionClosedContentNoAlertsExist(); + // Expand + await pageObjects.assetDetails.alertsSectionCollapsibleClick(); + await pageObjects.assetDetails.alertsSectionClosedContentNoAlertsMissing(); + }); + it('shows the CPU Profiling prompt if UI setting for Profiling integration is enabled', async () => { await setInfrastructureProfilingIntegrationUiSetting(true); await pageObjects.assetDetails.cpuProfilingPromptExists(); @@ -213,6 +222,42 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await setInfrastructureProfilingIntegrationUiSetting(false); await pageObjects.assetDetails.cpuProfilingPromptMissing(); }); + + describe('Alerts Section with alerts', () => { + before(async () => { + await navigateToNodeDetails('demo-stack-apache-01', 'demo-stack-apache-01'); + await pageObjects.header.waitUntilLoadingHasFinished(); + + await pageObjects.timePicker.setAbsoluteRange( + START_HOST_ALERTS_DATE.format(DATE_PICKER_FORMAT), + END_HOST_ALERTS_DATE.format(DATE_PICKER_FORMAT) + ); + + await pageObjects.assetDetails.clickOverviewTab(); + }); + + after(async () => { + await navigateToNodeDetails('Jennys-MBP.fritz.box', 'Jennys-MBP.fritz.box'); + await pageObjects.header.waitUntilLoadingHasFinished(); + + await pageObjects.timePicker.setAbsoluteRange( + START_HOST_PROCESSES_DATE.format(DATE_PICKER_FORMAT), + END_HOST_PROCESSES_DATE.format(DATE_PICKER_FORMAT) + ); + }); + + it('should show / hide alerts section with active alerts and show / hide closed section content', async () => { + await pageObjects.assetDetails.alertsSectionCollapsibleExist(); + // Expanded by default + await pageObjects.assetDetails.alertsSectionClosedContentMissing(); + // Collapse + await pageObjects.assetDetails.alertsSectionCollapsibleClick(); + await pageObjects.assetDetails.alertsSectionClosedContentExist(); + // Expand + await pageObjects.assetDetails.alertsSectionCollapsibleClick(); + await pageObjects.assetDetails.alertsSectionClosedContentMissing(); + }); + }); }); describe('Metadata Tab', () => { diff --git a/x-pack/test/functional/page_objects/asset_details.ts b/x-pack/test/functional/page_objects/asset_details.ts index 5e1ea574f8a81..cd34d9c2ca10b 100644 --- a/x-pack/test/functional/page_objects/asset_details.ts +++ b/x-pack/test/functional/page_objects/asset_details.ts @@ -89,6 +89,24 @@ export function AssetDetailsProvider({ getService }: FtrProviderContext) { return await testSubjects.existOrFail('infraAssetDetailsMetricsCollapsible'); }, + async alertsSectionCollapsibleClick() { + return await testSubjects.click('infraAssetDetailsAlertsCollapsible'); + }, + + async alertsSectionClosedContentExist() { + return await testSubjects.existOrFail('infraAssetDetailsAlertsClosedContentWithAlerts'); + }, + async alertsSectionClosedContentMissing() { + return await testSubjects.missingOrFail('infraAssetDetailsAlertsClosedContentWithAlerts'); + }, + + async alertsSectionClosedContentNoAlertsExist() { + return await testSubjects.existOrFail('infraAssetDetailsAlertsClosedContentNoAlerts'); + }, + async alertsSectionClosedContentNoAlertsMissing() { + return await testSubjects.missingOrFail('infraAssetDetailsAlertsClosedContentNoAlerts'); + }, + // Metadata async clickMetadataTab() { return testSubjects.click('infraAssetDetailsMetadataTab'); diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts b/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts index 3aaaf982c3597..aead4e6276c56 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts @@ -65,7 +65,8 @@ export class LlmProxy { } } - throw new Error('No interceptors found to handle request'); + response.writeHead(500, 'No interceptors found to handle request: ' + request.url); + response.end(); }) .listen(port); } @@ -111,7 +112,7 @@ export class LlmProxy { }), next: (msg) => { const chunk = createOpenAiChunk(msg); - return write(`data: ${JSON.stringify(chunk)}\n`); + return write(`data: ${JSON.stringify(chunk)}\n\n`); }, rawWrite: (chunk: string) => { return write(chunk); @@ -120,11 +121,11 @@ export class LlmProxy { await end(); }, complete: async () => { - await write('data: [DONE]'); + await write('data: [DONE]\n\n'); await end(); }, error: async (error) => { - await write(`data: ${JSON.stringify({ error })}`); + await write(`data: ${JSON.stringify({ error })}\n\n`); await end(); }, }; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts index c56af40a6ab29..82ad5b6dd1224 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts @@ -104,7 +104,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const chunk = JSON.stringify(createOpenAiChunk('Hello')); await simulator.rawWrite(`data: ${chunk.substring(0, 10)}`); - await simulator.rawWrite(`${chunk.substring(10)}\n`); + await simulator.rawWrite(`${chunk.substring(10)}\n\n`); await simulator.complete(); await new Promise<void>((resolve) => passThrough.on('end', () => resolve())); @@ -146,15 +146,17 @@ export default function ApiTest({ getService }: FtrProviderContext) { const titleInterceptor = proxy.intercept( 'title', (body) => - (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming).messages - .length === 1 + ( + JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming + ).functions?.find((fn) => fn.name === 'title_conversation') !== undefined ); const conversationInterceptor = proxy.intercept( 'conversation', (body) => - (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming).messages - .length !== 1 + ( + JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming + ).functions?.find((fn) => fn.name === 'title_conversation') === undefined ); const responsePromise = new Promise<Response>((resolve, reject) => { diff --git a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts index ce9fc050b5e09..c0b2b36dfc029 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts @@ -148,15 +148,17 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte const titleInterceptor = proxy.intercept( 'title', (body) => - (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming) - .messages.length === 1 + ( + JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming + ).functions?.find((fn) => fn.name === 'title_conversation') !== undefined ); const conversationInterceptor = proxy.intercept( 'conversation', (body) => - (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming) - .messages.length !== 1 + ( + JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming + ).functions?.find((fn) => fn.name === 'title_conversation') === undefined ); await testSubjects.setValue(ui.pages.conversations.chatInput, 'hello'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts index acb38b4dcefff..a080c4494833f 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts @@ -63,8 +63,8 @@ describe('Enrichment', { tags: ['@ess', '@serverless'] }, () => { }); it('Should has enrichment fields from legacy risk', function () { - cy.get(HOST_RISK_HEADER_COLUMN).contains('host.risk.calculated_level'); - cy.get(USER_RISK_HEADER_COLUMN).contains('user.risk.calculated_level'); + cy.get(HOST_RISK_HEADER_COLUMN).contains('Host Risk Level'); + cy.get(USER_RISK_HEADER_COLUMN).contains('User Risk Level'); scrollAlertTableColumnIntoView(HOST_RISK_COLUMN); cy.get(HOST_RISK_COLUMN).contains('Low'); scrollAlertTableColumnIntoView(USER_RISK_COLUMN); @@ -103,8 +103,8 @@ describe('Enrichment', { tags: ['@ess', '@serverless'] }, () => { }); it('Should has enrichment fields from legacy risk', function () { - cy.get(HOST_RISK_HEADER_COLUMN).contains('host.risk.calculated_level'); - cy.get(USER_RISK_HEADER_COLUMN).contains('user.risk.calculated_level'); + cy.get(HOST_RISK_HEADER_COLUMN).contains('Host Risk Level'); + cy.get(USER_RISK_HEADER_COLUMN).contains('User Risk Level'); scrollAlertTableColumnIntoView(HOST_RISK_COLUMN); cy.get(HOST_RISK_COLUMN).contains('Critical'); scrollAlertTableColumnIntoView(USER_RISK_COLUMN); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts index 0b94803bcd765..82fd6dd057ae5 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts @@ -23,6 +23,7 @@ export default function ({ getService }: FtrProviderContext) { let updateTemplate: typeof indexManagementService['templates']['api']['updateTemplate']; let deleteTemplates: typeof indexManagementService['templates']['api']['deleteTemplates']; let simulateTemplate: typeof indexManagementService['templates']['api']['simulateTemplate']; + let cleanUpTemplates: typeof indexManagementService['templates']['api']['cleanUpTemplates']; let getRandomString: () => string; describe('Index templates', function () { @@ -30,12 +31,22 @@ export default function ({ getService }: FtrProviderContext) { ({ templates: { helpers: { getTemplatePayload, catTemplate, getSerializedTemplate }, - api: { createTemplate, updateTemplate, deleteTemplates, simulateTemplate }, + api: { + createTemplate, + updateTemplate, + deleteTemplates, + simulateTemplate, + cleanUpTemplates, + }, }, } = indexManagementService); getRandomString = () => randomness.string({ casing: 'lower', alpha: true }); }); + after(async () => { + await cleanUpTemplates({ 'x-elastic-internal-origin': 'xxx' }); + }); + describe('get', () => { let templateName: string; @@ -93,6 +104,7 @@ export default function ({ getService }: FtrProviderContext) { 'hasMappings', '_kbnMeta', 'composedOf', + 'ignoreMissingComponentTemplates', ].sort(); expect(Object.keys(indexTemplateFound).sort()).to.eql(expectedKeys); @@ -113,6 +125,7 @@ export default function ({ getService }: FtrProviderContext) { 'template', '_kbnMeta', 'composedOf', + 'ignoreMissingComponentTemplates', ].sort(); expect(body.name).to.eql(templateName); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts b/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts index 4c8353487adce..da71d2ad4858a 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts @@ -11,6 +11,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const svlCommonApi = getService('svlCommonApi'); const supertest = getService('supertest'); + const config = getService('config'); describe('security/authentication', function () { describe('route access', () => { @@ -144,8 +145,7 @@ export default function ({ getService }: FtrProviderContext) { metadata: {}, operator: true, roles: ['superuser'], - // We use `elastic` for MKI, and `elastic_serverless` for any other testing environment. - username: expect.stringContaining('elastic'), + username: config.get('servers.kibana.username'), }); expect(status).toBe(200); }); diff --git a/yarn.lock b/yarn.lock index 62b6925bfe980..42e12233631a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7750,14 +7750,32 @@ "@types/node" ">=18.0.0" axios "^1.6.0" -"@smithy/eventstream-codec@^2.0.12": - version "2.0.12" - resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-2.0.12.tgz#99fab750d0ac3941f341d912d3c3a1ab985e1a7a" - integrity sha512-ZZQLzHBJkbiAAdj2C5K+lBlYp/XJ+eH2uy+jgJgYIFW/o5AM59Hlj7zyI44/ZTDIQWmBxb3EFv/c5t44V8/g8A== +"@smithy/eventstream-codec@^2.0.12", "@smithy/eventstream-codec@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-2.1.1.tgz#4405ab0f9c77d439c575560c4886e59ee17d6d38" + integrity sha512-E8KYBxBIuU4c+zrpR22VsVrOPoEDzk35bQR3E+xm4k6Pa6JqzkDOdMyf9Atac5GPNKHJBdVaQ4JtjdWX2rl/nw== dependencies: "@aws-crypto/crc32" "3.0.0" - "@smithy/types" "^2.4.0" - "@smithy/util-hex-encoding" "^2.0.0" + "@smithy/types" "^2.9.1" + "@smithy/util-hex-encoding" "^2.1.1" + tslib "^2.5.0" + +"@smithy/eventstream-serde-node@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-node/-/eventstream-serde-node-2.1.1.tgz#2e1afa27f9c7eb524c1c53621049c5e4e3cea6a5" + integrity sha512-LF882q/aFidFNDX7uROAGxq3H0B7rjyPkV6QDn6/KDQ+CG7AFkRccjxRf1xqajq/Pe4bMGGr+VKAaoF6lELIQw== + dependencies: + "@smithy/eventstream-serde-universal" "^2.1.1" + "@smithy/types" "^2.9.1" + tslib "^2.5.0" + +"@smithy/eventstream-serde-universal@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-2.1.1.tgz#0f5eec9ad033017973a67bafb5549782499488d2" + integrity sha512-LR0mMT+XIYTxk4k2fIxEA1BPtW3685QlqufUEUAX1AJcfFfxNDKEvuCRZbO8ntJb10DrIFVJR9vb0MhDCi0sAQ== + dependencies: + "@smithy/eventstream-codec" "^2.1.1" + "@smithy/types" "^2.9.1" tslib "^2.5.0" "@smithy/is-array-buffer@^2.0.0": @@ -7767,10 +7785,10 @@ dependencies: tslib "^2.5.0" -"@smithy/types@^2.4.0": - version "2.4.0" - resolved "https://registry.yarnpkg.com/@smithy/types/-/types-2.4.0.tgz#ed35e429e3ea3d089c68ed1bf951d0ccbdf2692e" - integrity sha512-iH1Xz68FWlmBJ9vvYeHifVMWJf82ONx+OybPW8ZGf5wnEv2S0UXcU4zwlwJkRXuLKpcSLHrraHbn2ucdVXLb4g== +"@smithy/types@^2.4.0", "@smithy/types@^2.9.1": + version "2.9.1" + resolved "https://registry.yarnpkg.com/@smithy/types/-/types-2.9.1.tgz#ed04d4144eed3b8bd26d20fc85aae8d6e357ebb9" + integrity sha512-vjXlKNXyprDYDuJ7UW5iobdmyDm6g8dDG+BFUncAg/3XJaN45Gy5RWWWUVgrzIK7S4R1KWgIX5LeJcfvSI24bw== dependencies: tslib "^2.5.0" @@ -7782,10 +7800,10 @@ "@smithy/is-array-buffer" "^2.0.0" tslib "^2.5.0" -"@smithy/util-hex-encoding@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@smithy/util-hex-encoding/-/util-hex-encoding-2.0.0.tgz#0aa3515acd2b005c6d55675e377080a7c513b59e" - integrity sha512-c5xY+NUnFqG6d7HFh1IFfrm3mGl29lC+vF+geHv4ToiuJCBmIfzx6IeHLg+OgRdPFKDXIw6pvi+p3CsscaMcMA== +"@smithy/util-hex-encoding@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@smithy/util-hex-encoding/-/util-hex-encoding-2.1.1.tgz#978252b9fb242e0a59bae4ead491210688e0d15f" + integrity sha512-3UNdP2pkYUUBGEXzQI9ODTDK+Tcu1BlCyDBaRHwyxhA+8xLP8agEKQq4MGmpjqb4VQAjq9TwlCQX0kP6XDKYLg== dependencies: tslib "^2.5.0" @@ -9346,6 +9364,13 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.0.tgz#5fb2e536c1ae9bf35366eed879e827fa59ca41c2" integrity sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ== +"@types/event-stream@^4.0.5": + version "4.0.5" + resolved "https://registry.yarnpkg.com/@types/event-stream/-/event-stream-4.0.5.tgz#29f1be5f4c0de2e0312cf3b5f7146c975c08d918" + integrity sha512-pQ/RR/iuBW8K8WmwYaaC1nkZH0cHonNAIw6ktG8BCNrNuqNeERfBzNIAOq6Z7tvLzpjcMV02SZ5pxAekAYQpWA== + dependencies: + "@types/node" "*" + "@types/expect@^1.20.4": version "1.20.4" resolved "https://registry.yarnpkg.com/@types/expect/-/expect-1.20.4.tgz#8288e51737bf7e3ab5d7c77bfa695883745264e5" @@ -9390,6 +9415,11 @@ resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.0.tgz#cbb49815a5e1129d5f23836a98d65d93822409af" integrity sha512-dxdRrUov2HVTbSRFX+7xwUPlbGYVEZK6PrSqClg2QPos3PNe0bCajkDDkDeeC1znjSH03KOEqVbXpnJuWa2wgQ== +"@types/flat@^5.0.5": + version "5.0.5" + resolved "https://registry.yarnpkg.com/@types/flat/-/flat-5.0.5.tgz#2304df0b2b1e6dde50d81f029593e0a1bc2474d3" + integrity sha512-nPLljZQKSnac53KDUDzuzdRfGI0TDb5qPrb+SrQyN3MtdQrOnGsKniHN1iYZsJEBIVQve94Y6gNz22sgISZq+Q== + "@types/flot@^0.0.31": version "0.0.31" resolved "https://registry.yarnpkg.com/@types/flot/-/flot-0.0.31.tgz#0daca37c6c855b69a0a7e2e37dd0f84b3db8c8c1" @@ -16620,6 +16650,11 @@ events@^3.0.0, events@^3.2.0, events@^3.3.0: resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== +eventsource-parser@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/eventsource-parser/-/eventsource-parser-1.1.1.tgz#576f8bcf391c5e5ccdea817abd9ead36d1754247" + integrity sha512-3Ej2iLj6ZnX+5CMxqyUb8syl9yVZwcwm8IIMrOJlF7I51zxOOrRlU3zxSb/6hFbl03ts1ZxHAGJdWLZOLyKG7w== + evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02" @@ -17272,7 +17307,7 @@ flat-cache@^3.0.4: flatted "^3.1.0" rimraf "^3.0.2" -flat@^5.0.2: +flat@5, flat@^5.0.2: version "5.0.2" resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==