;
getAlertsTableDefaultAlertActions: (
props: P
@@ -403,7 +411,7 @@ export class Plugin
connectorServices: this.connectorServices!,
});
},
- getAddRuleFlyout: (props: Omit) => {
+ getAddRuleFlyout: (props) => {
return getAddRuleFlyoutLazy({
...props,
actionTypeRegistry: this.actionTypeRegistry,
@@ -411,9 +419,7 @@ export class Plugin
connectorServices: this.connectorServices!,
});
},
- getEditRuleFlyout: (
- props: Omit
- ) => {
+ getEditRuleFlyout: (props) => {
return getEditRuleFlyoutLazy({
...props,
actionTypeRegistry: this.actionTypeRegistry,
diff --git a/x-pack/plugins/triggers_actions_ui/public/types.ts b/x-pack/plugins/triggers_actions_ui/public/types.ts
index b47d80a0839e5..36cc294bbda5f 100644
--- a/x-pack/plugins/triggers_actions_ui/public/types.ts
+++ b/x-pack/plugins/triggers_actions_ui/public/types.ts
@@ -51,6 +51,7 @@ import {
AlertingFrameworkHealth,
RuleNotifyWhenType,
RuleTypeParams,
+ RuleTypeMetaData,
ActionVariable,
RuleLastRun,
MaintenanceWindow,
@@ -127,6 +128,7 @@ export type {
AlertingFrameworkHealth,
RuleNotifyWhenType,
RuleTypeParams,
+ RuleTypeMetaData,
ResolvedRule,
SanitizedRule,
RuleStatusDropdownProps,
@@ -412,8 +414,11 @@ export enum EditConnectorTabs {
Test = 'test',
}
-export interface RuleEditProps> {
- initialRule: Rule;
+export interface RuleEditProps<
+ Params extends RuleTypeParams = RuleTypeParams,
+ MetaData extends RuleTypeMetaData = RuleTypeMetaData
+> {
+ initialRule: Rule;
ruleTypeRegistry: RuleTypeRegistryContract;
actionTypeRegistry: ActionTypeRegistryContract;
onClose: (reason: RuleFlyoutCloseReason, metadata?: MetaData) => void;
@@ -425,14 +430,27 @@ export interface RuleEditProps> {
ruleType?: RuleType;
}
-export interface RuleAddProps> {
+export interface RuleAddProps<
+ Params extends RuleTypeParams = RuleTypeParams,
+ MetaData extends RuleTypeMetaData = RuleTypeMetaData
+> {
+ /**
+ * ID of the feature this rule should be created for.
+ *
+ * Notes:
+ * - The feature needs to be registered using `featuresPluginSetup.registerKibanaFeature()` API during your plugin's setup phase.
+ * - The user needs to have permission to access the feature in order to create the rule.
+ * */
consumer: string;
ruleTypeRegistry: RuleTypeRegistryContract;
actionTypeRegistry: ActionTypeRegistryContract;
onClose: (reason: RuleFlyoutCloseReason, metadata?: MetaData) => void;
ruleTypeId?: string;
+ /**
+ * Determines whether the user should be able to change the rule type in the UI.
+ */
canChangeTrigger?: boolean;
- initialValues?: Partial;
+ initialValues?: Partial>;
/** @deprecated use `onSave` as a callback after an alert is saved*/
reloadRules?: () => Promise;
hideGrouping?: boolean;
@@ -445,8 +463,8 @@ export interface RuleAddProps> {
useRuleProducer?: boolean;
initialSelectedConsumer?: RuleCreationValidConsumer | null;
}
-export interface RuleDefinitionProps {
- rule: Rule;
+export interface RuleDefinitionProps {
+ rule: Rule;
ruleTypeRegistry: RuleTypeRegistryContract;
actionTypeRegistry: ActionTypeRegistryContract;
onEditRule: () => Promise;
From 9591304b0d9900249dca6a39cb7fe7360f90666f Mon Sep 17 00:00:00 2001
From: Stratoula Kalafateli
Date: Fri, 9 Feb 2024 10:11:48 +0200
Subject: [PATCH 02/13] [Obs ai assistant] Use context to handle the multipane
flyout (#176373)
## Summary
Use a context to handle the multipane flyout properties that are needed
for the `visualize_esql` function.
### How to test
Should work exactly as before. The editing flyout should open when the
user clicks the pencil button on a lens embeddable
---
.../public/components/chat/chat_body.tsx | 9 +-
.../public/components/chat/chat_flyout.tsx | 220 +++++++++---------
.../public/components/chat/chat_timeline.tsx | 16 +-
.../public/components/render_function.tsx | 11 +-
...ai_assistant_multipane_flyout_provider.tsx | 16 ++
.../public/functions/visualize_esql.test.tsx | 24 +-
.../public/functions/visualize_esql.tsx | 20 +-
.../conversations/conversation_view.tsx | 4 -
.../public/service/create_chat_service.ts | 3 +-
.../public/types.ts | 5 +-
..._timeline_items_from_conversation.test.tsx | 7 +-
.../get_timeline_items_from_conversation.tsx | 5 +-
12 files changed, 157 insertions(+), 183 deletions(-)
create mode 100644 x-pack/plugins/observability_ai_assistant/public/context/observability_ai_assistant_multipane_flyout_provider.tsx
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx
index 8ed26d71acc58..a89419c366a2d 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx
@@ -35,11 +35,7 @@ import { ChatTimeline } from './chat_timeline';
import { Feedback } from '../feedback_buttons';
import { IncorrectLicensePanel } from './incorrect_license_panel';
import { WelcomeMessage } from './welcome_message';
-import {
- ChatActionClickHandler,
- ChatActionClickType,
- type ChatFlyoutSecondSlotHandler,
-} from './types';
+import { ChatActionClickHandler, ChatActionClickType } from './types';
import { ASSISTANT_SETUP_TITLE, EMPTY_CONVERSATION_TITLE, UPGRADE_LICENSE_TITLE } from '../../i18n';
import type { StartedFrom } from '../../utils/get_timeline_items_from_conversation';
import { TELEMETRY, sendEvent } from '../../analytics';
@@ -94,7 +90,6 @@ const animClassName = css`
const PADDING_AND_BORDER = 32;
export function ChatBody({
- chatFlyoutSecondSlotHandler,
connectors,
currentUser,
flyoutWidthMode,
@@ -107,7 +102,6 @@ export function ChatBody({
onConversationUpdate,
onToggleFlyoutWidthMode,
}: {
- chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler;
connectors: UseGenAIConnectorsResult;
currentUser?: Pick;
flyoutWidthMode?: FlyoutWidthMode;
@@ -362,7 +356,6 @@ export function ChatBody({
onStopGenerating={() => {
stop();
}}
- chatFlyoutSecondSlotHandler={chatFlyoutSecondSlotHandler}
onActionClick={handleActionClick}
/>
)}
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx
index 6823153397ca4..5a8b0ee3b3776 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_flyout.tsx
@@ -18,6 +18,7 @@ import {
EuiToolTip,
useEuiTheme,
} from '@elastic/eui';
+import { ObservabilityAIAssistantMultipaneFlyoutProvider } from '../../context/observability_ai_assistant_multipane_flyout_provider';
import { useForceUpdate } from '../../hooks/use_force_update';
import { useCurrentUser } from '../../hooks/use_current_user';
import { useGenAIConnectors } from '../../hooks/use_genai_connectors';
@@ -134,133 +135,136 @@ export function ChatFlyout({
};
return isOpen ? (
- {
- onClose();
- setIsSecondSlotVisible(false);
- if (secondSlotContainer) {
- ReactDOM.unmountComponentAtNode(secondSlotContainer);
- }
+
-
-
-
- setConversationsExpanded(!conversationsExpanded)}
- />
-
- }
- />
-
- {conversationsExpanded ? (
-
- ) : (
+ {
+ onClose();
+ setIsSecondSlotVisible(false);
+ if (secondSlotContainer) {
+ ReactDOM.unmountComponentAtNode(secondSlotContainer);
+ }
+ }}
+ >
+
+
setConversationsExpanded(!conversationsExpanded)}
/>
}
- className={newChatButtonClassName}
/>
- )}
-
-
- {
- setConversationId(conversation.conversation.id);
- }}
- onToggleFlyoutWidthMode={handleToggleFlyoutWidthMode}
- />
-
+ {conversationsExpanded ? (
+
+ ) : (
+
+
+
+ }
+ className={newChatButtonClassName}
+ />
+ )}
+
-
-
+ {
+ setConversationId(conversation.conversation.id);
+ }}
+ onToggleFlyoutWidthMode={handleToggleFlyoutWidthMode}
+ />
+
+
+
-
-
-
+ >
+
+
+
+
+
) : null;
}
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx
index 48cf4070b0b96..0baccaf979f1f 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx
@@ -13,7 +13,7 @@ import { omit } from 'lodash';
import type { Feedback } from '../feedback_buttons';
import type { Message } from '../../../common';
import type { UseKnowledgeBaseResult } from '../../hooks/use_knowledge_base';
-import type { ChatActionClickHandler, ChatFlyoutSecondSlotHandler } from './types';
+import type { ChatActionClickHandler } from './types';
import type { ObservabilityAIAssistantChatService } from '../../types';
import type { TelemetryEventTypeWithPayload } from '../../analytics';
import { ChatItem } from './chat_item';
@@ -54,7 +54,6 @@ export interface ChatTimelineProps {
chatState: ChatState;
currentUser?: Pick;
startedFrom?: StartedFrom;
- chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler;
onEdit: (message: Message, messageAfterEdit: Message) => void;
onFeedback: (message: Message, feedback: Feedback) => void;
onRegenerate: (message: Message) => void;
@@ -69,7 +68,6 @@ export function ChatTimeline({
hasConnector,
currentUser,
startedFrom,
- chatFlyoutSecondSlotHandler,
onEdit,
onFeedback,
onRegenerate,
@@ -86,7 +84,6 @@ export function ChatTimeline({
currentUser,
startedFrom,
chatState,
- chatFlyoutSecondSlotHandler,
onActionClick,
});
@@ -110,16 +107,7 @@ export function ChatTimeline({
}
return consolidatedChatItems;
- }, [
- chatService,
- hasConnector,
- messages,
- currentUser,
- startedFrom,
- chatState,
- chatFlyoutSecondSlotHandler,
- onActionClick,
- ]);
+ }, [chatService, hasConnector, messages, currentUser, startedFrom, chatState, onActionClick]);
return (
- {chatService.renderFunction(
- props.name,
- props.arguments,
- props.response,
- props.onActionClick,
- props.chatFlyoutSecondSlotHandler
- )}
+ {chatService.renderFunction(props.name, props.arguments, props.response, props.onActionClick)}
);
}
diff --git a/x-pack/plugins/observability_ai_assistant/public/context/observability_ai_assistant_multipane_flyout_provider.tsx b/x-pack/plugins/observability_ai_assistant/public/context/observability_ai_assistant_multipane_flyout_provider.tsx
new file mode 100644
index 0000000000000..93a091ff4a7d4
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/public/context/observability_ai_assistant_multipane_flyout_provider.tsx
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createContext } from 'react';
+import type { ChatFlyoutSecondSlotHandler } from '../types';
+
+export const ObservabilityAIAssistantMultipaneFlyoutContext = createContext<
+ ChatFlyoutSecondSlotHandler | undefined
+>(undefined);
+
+export const ObservabilityAIAssistantMultipaneFlyoutProvider =
+ ObservabilityAIAssistantMultipaneFlyoutContext.Provider;
diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx
index 789697fbaaaa8..de7c4f04f241c 100644
--- a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.test.tsx
@@ -12,6 +12,7 @@ import type { LensPublicStart } from '@kbn/lens-plugin/public';
import { lensPluginMock } from '@kbn/lens-plugin/public/mocks/lens_plugin_mock';
import { uiActionsPluginMock } from '@kbn/ui-actions-plugin/public/mocks';
import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
+import { ObservabilityAIAssistantMultipaneFlyoutProvider } from '../context/observability_ai_assistant_multipane_flyout_provider';
import { VisualizeESQL } from './visualize_esql';
describe('VisualizeESQL', () => {
@@ -50,19 +51,22 @@ describe('VisualizeESQL', () => {
},
] as DatatableColumn[];
render(
-
+ >
+
+
);
}
diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx
index 05145c6130b4f..61295f4faf6f7 100644
--- a/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/functions/visualize_esql.tsx
@@ -22,7 +22,7 @@ import type {
TypedLensByValueInput,
InlineEditLensEmbeddableContext,
} from '@kbn/lens-plugin/public';
-import React, { useState, useEffect, useCallback, useMemo } from 'react';
+import React, { useState, useEffect, useCallback, useMemo, useContext } from 'react';
import ReactDOM from 'react-dom';
import useAsync from 'react-use/lib/useAsync';
import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
@@ -30,17 +30,14 @@ import {
VisualizeESQLFunctionArguments,
VisualizeESQLUserIntention,
} from '../../common/functions/visualize_esql';
+import { ObservabilityAIAssistantMultipaneFlyoutContext } from '../context/observability_ai_assistant_multipane_flyout_provider';
import type {
ObservabilityAIAssistantPluginStartDependencies,
ObservabilityAIAssistantService,
RegisterRenderFunctionDefinition,
RenderFunction,
} from '../types';
-import {
- type ChatActionClickHandler,
- ChatActionClickType,
- ChatFlyoutSecondSlotHandler,
-} from '../components/chat/types';
+import { type ChatActionClickHandler, ChatActionClickType } from '../components/chat/types';
interface VisualizeLensResponse {
content: DatatableColumn[];
@@ -63,12 +60,6 @@ interface VisualizeESQLProps {
* If not given, the embeddable gets them from the suggestions api
*/
userOverrides?: unknown;
- /** Optional, should be passed if the embeddable is rendered in a flyout
- * If not given, the inline editing push flyout won't open
- * The code will be significantly improved,
- * if this is addressed https://github.com/elastic/eui/issues/7443
- */
- chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler;
/** User's preferation chart type as it comes from the model */
preferredChartType?: string;
}
@@ -85,7 +76,6 @@ export function VisualizeESQL({
query,
onActionClick,
userOverrides,
- chatFlyoutSecondSlotHandler,
preferredChartType,
}: VisualizeESQLProps) {
// fetch the pattern from the query
@@ -100,6 +90,8 @@ export function VisualizeESQL({
});
}, [indexPattern]);
+ const chatFlyoutSecondSlotHandler = useContext(ObservabilityAIAssistantMultipaneFlyoutContext);
+
const [isSaveModalOpen, setIsSaveModalOpen] = useState(false);
const [lensInput, setLensInput] = useState(
userOverrides as TypedLensByValueInput
@@ -316,7 +308,6 @@ export function registerVisualizeQueryRenderFunction({
arguments: { query, userOverrides, intention },
response,
onActionClick,
- chatFlyoutSecondSlotHandler,
}: Parameters>[0]) => {
const { content } = response as VisualizeLensResponse;
@@ -370,7 +361,6 @@ export function registerVisualizeQueryRenderFunction({
query={query}
onActionClick={onActionClick}
userOverrides={userOverrides}
- chatFlyoutSecondSlotHandler={chatFlyoutSecondSlotHandler}
preferredChartType={preferredChartType}
/>
);
diff --git a/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx b/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx
index 4620c0cf2775d..0f49389c1c60d 100644
--- a/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/routes/conversations/conversation_view.tsx
@@ -199,10 +199,6 @@ export function ConversationView() {
showLinkToConversationsApp={false}
startedFrom="conversationView"
onConversationUpdate={handleConversationUpdate}
- chatFlyoutSecondSlotHandler={{
- container: secondSlotContainer,
- setVisibility: setIsSecondSlotVisible,
- }}
/>
diff --git a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
index 211f25b045b77..17ea46c56681e 100644
--- a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
@@ -116,7 +116,7 @@ export async function createChatService({
return {
analytics,
- renderFunction: (name, args, response, onActionClick, chatFlyoutSecondSlotHandler) => {
+ renderFunction: (name, args, response, onActionClick) => {
const fn = renderFunctionRegistry.get(name);
if (!fn) {
@@ -134,7 +134,6 @@ export async function createChatService({
response: parsedResponse,
arguments: parsedArguments,
onActionClick,
- chatFlyoutSecondSlotHandler,
});
},
getContexts: () => contextDefinitions,
diff --git a/x-pack/plugins/observability_ai_assistant/public/types.ts b/x-pack/plugins/observability_ai_assistant/public/types.ts
index 418c7eca16b19..e303b01a5c9e9 100644
--- a/x-pack/plugins/observability_ai_assistant/public/types.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/types.ts
@@ -49,6 +49,7 @@ import type { UseGenAIConnectorsResult } from './hooks/use_genai_connectors';
export type { CreateChatCompletionResponseChunk } from '../common/types';
export type { PendingMessage };
+export type { ChatFlyoutSecondSlotHandler };
export interface ObservabilityAIAssistantChatService {
analytics: AnalyticsServiceStart;
@@ -76,8 +77,7 @@ export interface ObservabilityAIAssistantChatService {
name: string,
args: string | undefined,
response: { data?: string; content?: string },
- onActionClick: ChatActionClickHandler,
- chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler
+ onActionClick: ChatActionClickHandler
) => React.ReactNode;
}
@@ -95,7 +95,6 @@ export type RenderFunction
= (op
arguments: TArguments;
response: TResponse;
onActionClick: ChatActionClickHandler;
- chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler;
}) => React.ReactNode;
export type RegisterRenderFunctionDefinition<
diff --git a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx
index 600256d66a7bc..8135111a6f548 100644
--- a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx
@@ -231,10 +231,6 @@ describe('getTimelineItemsFromConversation', () => {
},
],
onActionClick: jest.fn(),
- chatFlyoutSecondSlotHandler: {
- container: null,
- setVisibility: jest.fn(),
- },
});
});
@@ -270,8 +266,7 @@ describe('getTimelineItemsFromConversation', () => {
'my_render_function',
JSON.stringify({ foo: 'bar' }),
{ content: '[]', name: 'my_render_function', role: 'user' },
- expect.any(Function),
- { container: null, setVisibility: expect.any(Function) }
+ expect.any(Function)
);
expect(container.textContent).toEqual('Rendered');
diff --git a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx
index 40b54708e5b6c..d1f14e30d6097 100644
--- a/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx
@@ -17,7 +17,7 @@ import { RenderFunction } from '../components/render_function';
import type { ObservabilityAIAssistantChatService } from '../types';
import { ChatState } from '../hooks/use_chat';
import { safeJsonParse } from './safe_json_parse';
-import type { ChatActionClickHandler, ChatFlyoutSecondSlotHandler } from '../components/chat/types';
+import type { ChatActionClickHandler } from '../components/chat/types';
function convertMessageToMarkdownCodeBlock(message: Message['message']) {
let value: object;
@@ -65,7 +65,6 @@ export function getTimelineItemsfromConversation({
messages,
startedFrom,
chatState,
- chatFlyoutSecondSlotHandler,
onActionClick,
}: {
chatService: ObservabilityAIAssistantChatService;
@@ -74,7 +73,6 @@ export function getTimelineItemsfromConversation({
messages: Message[];
startedFrom?: StartedFrom;
chatState: ChatState;
- chatFlyoutSecondSlotHandler?: ChatFlyoutSecondSlotHandler;
onActionClick: ChatActionClickHandler;
}): ChatTimelineItem[] {
const messagesWithoutSystem = messages.filter(
@@ -169,7 +167,6 @@ export function getTimelineItemsfromConversation({
arguments={prevFunctionCall?.arguments}
response={message.message}
onActionClick={onActionClick}
- chatFlyoutSecondSlotHandler={chatFlyoutSecondSlotHandler}
/>
) : undefined;
From 736af7b0e07512b7797e3759d076dbcebbb55b64 Mon Sep 17 00:00:00 2001
From: Robert Oskamp
Date: Fri, 9 Feb 2024 09:15:18 +0100
Subject: [PATCH 03/13] [FTR] Fix URL checks in navigateToApp (#176546)
## Summary
This PR fixes the URL check for successful navigation in the `common`
PageObject `navigateToApp` method.
---
test/functional/page_objects/common_page.ts | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/test/functional/page_objects/common_page.ts b/test/functional/page_objects/common_page.ts
index 653b0213bc430..98ac4d0abfe04 100644
--- a/test/functional/page_objects/common_page.ts
+++ b/test/functional/page_objects/common_page.ts
@@ -332,14 +332,16 @@ export class CommonPageObject extends FtrService {
}
currentUrl = (await this.browser.getCurrentUrl()).replace(/\/\/\w+:\w+@/, '//');
+ const decodedAppUrl = decodeURIComponent(appUrl);
+ const decodedCurrentUrl = decodeURIComponent(currentUrl);
- const navSuccessful = currentUrl
+ const navSuccessful = decodedCurrentUrl
.replace(':80/', '/')
.replace(':443/', '/')
- .startsWith(appUrl.replace(':80/', '/').replace(':443/', '/'));
+ .startsWith(decodedAppUrl.replace(':80/', '/').replace(':443/', '/'));
if (!navSuccessful) {
- const msg = `App failed to load: ${appName} in ${this.defaultFindTimeout}ms appUrl=${appUrl} currentUrl=${currentUrl}`;
+ const msg = `App failed to load: ${appName} in ${this.defaultFindTimeout}ms appUrl=${decodedAppUrl} currentUrl=${decodedCurrentUrl}`;
this.log.debug(msg);
throw new Error(msg);
}
From 44df1f4caad795a3c5be45774520c1c6b3dcac22 Mon Sep 17 00:00:00 2001
From: Dario Gieselaar
Date: Fri, 9 Feb 2024 09:17:20 +0100
Subject: [PATCH 04/13] [Obs AI Assistant] Bedrock/Claude support (#176191)
~This PR still needs work (tests, mainly), so keeping it in draft for
now, but feel free to take it for a spin.~
Implements Bedrock support, specifically for the Claude models.
Architecturally, this introduces LLM adapters: one for OpenAI (which is
what we already have), and one for Bedrock/Claude. The Bedrock/Claude
adapter does the following things:
- parses data from a SerDe (an AWS concept IIUC) stream using
`@smithy/eventstream-serde-node`.
- Converts function requests and results into XML and back (to some
extent)
- some slight changes to existing functionality to achieve _some_ kind
of baseline performance with Bedrock + Claude.
Generally, GPT seems better at implicit tasks. Claude needs explicit
tasks, otherwise it will take things too literally. For instance, I had
to use a function for generating a title because Claude was too eager to
add explanations. For the `classify_esql` function, I had to add extra
instructions to stop it from requesting information that is not there.
It is prone to generating invalid XML.
---------
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
---
.../pre-configured-connectors.asciidoc | 2 +-
docs/settings/alert-action-settings.asciidoc | 2 +-
package.json | 6 +
packages/kbn-test/jest-preset.js | 2 +-
.../jest_integration_node/jest-preset.js | 2 +-
.../plugins/actions/docs/openapi/bundled.json | 4 +-
.../plugins/actions/docs/openapi/bundled.yaml | 2 +-
.../docs/openapi/bundled_serverless.json | 4 +-
.../docs/openapi/bundled_serverless.yaml | 2 +-
.../schemas/config_properties_bedrock.yaml | 2 +-
.../common/connectors.ts | 25 ++
.../common/conversation_complete.ts | 17 +-
.../common/utils/process_openai_stream.ts | 1 -
.../observability_ai_assistant/kibana.jsonc | 4 +-
.../components/chat/welcome_message.tsx | 3 +-
.../public/service/create_chat_service.ts | 16 +-
.../scripts/evaluation/README.md | 2 +-
.../scripts/evaluation/kibana_client.ts | 22 +-
.../server/functions/index.ts | 3 +-
.../server/functions/query/index.ts | 106 ++++----
.../server/functions/recall.ts | 14 +-
.../server/routes/chat/route.ts | 22 +-
.../server/routes/connectors/route.ts | 3 +-
.../adapters/bedrock_claude_adapter.test.ts | 239 ++++++++++++++++
.../client/adapters/bedrock_claude_adapter.ts | 228 ++++++++++++++++
.../service/client/adapters/openai_adapter.ts | 69 +++++
.../adapters/process_bedrock_stream.test.ts | 256 ++++++++++++++++++
.../client/adapters/process_bedrock_stream.ts | 151 +++++++++++
.../server/service/client/adapters/types.ts | 25 ++
.../server/service/client/index.test.ts | 27 +-
.../server/service/client/index.ts | 206 +++++++-------
..._deserialized_xml_with_json_schema.test.ts | 128 +++++++++
...nvert_deserialized_xml_with_json_schema.ts | 106 ++++++++
.../eventsource_stream_into_observable.ts | 38 +++
.../util/eventstream_serde_into_observable.ts | 58 ++++
.../server/service/util/flush_buffer.ts | 63 +++++
.../json_schema_to_flat_parameters.test.ts | 208 ++++++++++++++
.../util/json_schema_to_flat_parameters.ts | 73 +++++
.../service/util/observable_into_stream.ts | 5 +-
.../service/util/stream_into_observable.ts | 31 ++-
.../server/types.ts | 5 +-
.../observability_ai_assistant/tsconfig.json | 2 +
.../common/bedrock/constants.ts | 2 +-
.../stack_connectors/common/bedrock/schema.ts | 2 +
.../public/connector_types/bedrock/params.tsx | 2 +-
.../connector_types/bedrock/bedrock.test.ts | 58 +++-
.../server/connector_types/bedrock/bedrock.ts | 48 +++-
.../server/connector_types/bedrock/index.ts | 10 +-
.../server/bedrock_simulation.ts | 2 +-
.../tests/actions/connector_types/bedrock.ts | 12 +-
.../common/create_llm_proxy.ts | 9 +-
.../tests/complete/complete.spec.ts | 12 +-
.../tests/conversations/index.spec.ts | 10 +-
yarn.lock | 65 ++++-
54 files changed, 2159 insertions(+), 257 deletions(-)
create mode 100644 x-pack/plugins/observability_ai_assistant/common/connectors.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.test.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/client/adapters/openai_adapter.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.test.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/client/adapters/types.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.test.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/eventsource_stream_into_observable.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/eventstream_serde_into_observable.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/flush_buffer.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.test.ts
create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.ts
diff --git a/docs/management/connectors/pre-configured-connectors.asciidoc b/docs/management/connectors/pre-configured-connectors.asciidoc
index b7293b6232190..c027220376cdf 100644
--- a/docs/management/connectors/pre-configured-connectors.asciidoc
+++ b/docs/management/connectors/pre-configured-connectors.asciidoc
@@ -148,7 +148,7 @@ xpack.actions.preconfigured:
actionTypeId: .bedrock
config:
apiUrl: https://bedrock-runtime.us-east-1.amazonaws.com <1>
- defaultModel: anthropic.claude-v2 <2>
+ defaultModel: anthropic.claude-v2:1 <2>
secrets:
accessKey: key-value <3>
secret: secret-value <4>
diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc
index b7d7e8d344a32..2bfde478a494d 100644
--- a/docs/settings/alert-action-settings.asciidoc
+++ b/docs/settings/alert-action-settings.asciidoc
@@ -340,7 +340,7 @@ For a <>, specifies a string f
The default model to use for requests, which varies by connector:
+
--
-* For an <>, current support is for the Anthropic Claude models. Defaults to `anthropic.claude-v2`.
+* For an <>, current support is for the Anthropic Claude models. Defaults to `anthropic.claude-v2:1`.
* For a <>, it is optional and applicable only when `xpack.actions.preconfigured..config.apiProvider` is `OpenAI`.
--
diff --git a/package.json b/package.json
index 05b09aa56196c..c11b0b349099f 100644
--- a/package.json
+++ b/package.json
@@ -881,6 +881,8 @@
"@reduxjs/toolkit": "1.9.7",
"@slack/webhook": "^7.0.1",
"@smithy/eventstream-codec": "^2.0.12",
+ "@smithy/eventstream-serde-node": "^2.1.1",
+ "@smithy/types": "^2.9.1",
"@smithy/util-utf8": "^2.0.0",
"@tanstack/react-query": "^4.29.12",
"@tanstack/react-query-devtools": "^4.29.12",
@@ -946,6 +948,7 @@
"diff": "^5.1.0",
"elastic-apm-node": "^4.4.0",
"email-addresses": "^5.0.0",
+ "eventsource-parser": "^1.1.1",
"execa": "^5.1.1",
"expiry-js": "0.1.7",
"exponential-backoff": "^3.1.1",
@@ -954,6 +957,7 @@
"fast-glob": "^3.3.2",
"fflate": "^0.6.9",
"file-saver": "^1.3.8",
+ "flat": "5",
"fnv-plus": "^1.3.1",
"font-awesome": "4.7.0",
"formik": "^2.4.5",
@@ -1380,11 +1384,13 @@
"@types/ejs": "^3.0.6",
"@types/enzyme": "^3.10.12",
"@types/eslint": "^8.44.2",
+ "@types/event-stream": "^4.0.5",
"@types/express": "^4.17.13",
"@types/extract-zip": "^1.6.2",
"@types/faker": "^5.1.5",
"@types/fetch-mock": "^7.3.1",
"@types/file-saver": "^2.0.0",
+ "@types/flat": "^5.0.5",
"@types/flot": "^0.0.31",
"@types/fnv-plus": "^1.3.0",
"@types/geojson": "^7946.0.10",
diff --git a/packages/kbn-test/jest-preset.js b/packages/kbn-test/jest-preset.js
index df9ed4cab4f51..de4e6032ba52f 100644
--- a/packages/kbn-test/jest-preset.js
+++ b/packages/kbn-test/jest-preset.js
@@ -105,7 +105,7 @@ module.exports = {
transformIgnorePatterns: [
// ignore all node_modules except monaco-editor, monaco-yaml and react-monaco-editor which requires babel transforms to handle dynamic import()
// since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842)
- '[/\\\\]node_modules(?)[/\\\\].+\\.js$',
+ '[/\\\\]node_modules(?)[/\\\\].+\\.js$',
'packages/kbn-pm/dist/index.js',
'[/\\\\]node_modules(?)/dist/[/\\\\].+\\.js$',
'[/\\\\]node_modules(?)/dist/util/[/\\\\].+\\.js$',
diff --git a/packages/kbn-test/jest_integration_node/jest-preset.js b/packages/kbn-test/jest_integration_node/jest-preset.js
index 631b2c4f9350e..6472237c5dd17 100644
--- a/packages/kbn-test/jest_integration_node/jest-preset.js
+++ b/packages/kbn-test/jest_integration_node/jest-preset.js
@@ -22,7 +22,7 @@ module.exports = {
// An array of regexp pattern strings that are matched against, matched files will skip transformation:
transformIgnorePatterns: [
// since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842)
- '[/\\\\]node_modules(?)[/\\\\].+\\.js$',
+ '[/\\\\]node_modules(?)[/\\\\].+\\.js$',
'[/\\\\]node_modules(?)/dist/[/\\\\].+\\.js$',
'[/\\\\]node_modules(?)/dist/util/[/\\\\].+\\.js$',
],
diff --git a/x-pack/plugins/actions/docs/openapi/bundled.json b/x-pack/plugins/actions/docs/openapi/bundled.json
index d165392087670..d910d5ad6501e 100644
--- a/x-pack/plugins/actions/docs/openapi/bundled.json
+++ b/x-pack/plugins/actions/docs/openapi/bundled.json
@@ -2240,7 +2240,7 @@
"defaultModel": {
"type": "string",
"description": "The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models.\n",
- "default": "anthropic.claude-v2"
+ "default": "anthropic.claude-v2:1"
}
}
},
@@ -6841,4 +6841,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugins/actions/docs/openapi/bundled.yaml b/x-pack/plugins/actions/docs/openapi/bundled.yaml
index 58ea32fe25764..cd55a90afa483 100644
--- a/x-pack/plugins/actions/docs/openapi/bundled.yaml
+++ b/x-pack/plugins/actions/docs/openapi/bundled.yaml
@@ -1498,7 +1498,7 @@ components:
type: string
description: |
The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models.
- default: anthropic.claude-v2
+ default: anthropic.claude-v2:1
secrets_properties_bedrock:
title: Connector secrets properties for an Amazon Bedrock connector
description: Defines secrets for connectors when type is `.bedrock`.
diff --git a/x-pack/plugins/actions/docs/openapi/bundled_serverless.json b/x-pack/plugins/actions/docs/openapi/bundled_serverless.json
index acde35b764a5e..ba7d2b16be139 100644
--- a/x-pack/plugins/actions/docs/openapi/bundled_serverless.json
+++ b/x-pack/plugins/actions/docs/openapi/bundled_serverless.json
@@ -1226,7 +1226,7 @@
"defaultModel": {
"type": "string",
"description": "The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models.\n",
- "default": "anthropic.claude-v2"
+ "default": "anthropic.claude-v2:1"
}
}
},
@@ -4377,4 +4377,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml b/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml
index 3d9be12c8077e..564b121ec663b 100644
--- a/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml
+++ b/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml
@@ -857,7 +857,7 @@ components:
type: string
description: |
The generative artificial intelligence model for Amazon Bedrock to use. Current support is for the Anthropic Claude models.
- default: anthropic.claude-v2
+ default: anthropic.claude-v2:1
secrets_properties_bedrock:
title: Connector secrets properties for an Amazon Bedrock connector
description: Defines secrets for connectors when type is `.bedrock`.
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml
index 25b279c423739..189a5d5e2e05e 100644
--- a/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_bedrock.yaml
@@ -12,4 +12,4 @@ properties:
description: >
The generative artificial intelligence model for Amazon Bedrock to use.
Current support is for the Anthropic Claude models.
- default: anthropic.claude-v2
\ No newline at end of file
+ default: anthropic.claude-v2:1
diff --git a/x-pack/plugins/observability_ai_assistant/common/connectors.ts b/x-pack/plugins/observability_ai_assistant/common/connectors.ts
new file mode 100644
index 0000000000000..2b834081a7ac9
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/common/connectors.ts
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export enum ObservabilityAIAssistantConnectorType {
+ Bedrock = '.bedrock',
+ OpenAI = '.gen-ai',
+}
+
+export const SUPPORTED_CONNECTOR_TYPES = [
+ ObservabilityAIAssistantConnectorType.OpenAI,
+ ObservabilityAIAssistantConnectorType.Bedrock,
+];
+
+export function isSupportedConnectorType(
+ type: string
+): type is ObservabilityAIAssistantConnectorType {
+ return (
+ type === ObservabilityAIAssistantConnectorType.Bedrock ||
+ type === ObservabilityAIAssistantConnectorType.OpenAI
+ );
+}
diff --git a/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts b/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts
index f5fe0d37408c2..b082478bba100 100644
--- a/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts
+++ b/x-pack/plugins/observability_ai_assistant/common/conversation_complete.ts
@@ -14,6 +14,7 @@ export enum StreamingChatResponseEventType {
ConversationUpdate = 'conversationUpdate',
MessageAdd = 'messageAdd',
ChatCompletionError = 'chatCompletionError',
+ BufferFlush = 'bufferFlush',
}
type StreamingChatResponseEventBase<
@@ -76,6 +77,13 @@ export type ChatCompletionErrorEvent = StreamingChatResponseEventBase<
}
>;
+export type BufferFlushEvent = StreamingChatResponseEventBase<
+ StreamingChatResponseEventType.BufferFlush,
+ {
+ data?: string;
+ }
+>;
+
export type StreamingChatResponseEvent =
| ChatCompletionChunkEvent
| ConversationCreateEvent
@@ -129,7 +137,14 @@ export function createConversationNotFoundError() {
);
}
-export function createInternalServerError(originalErrorMessage: string) {
+export function createInternalServerError(
+ originalErrorMessage: string = i18n.translate(
+ 'xpack.observabilityAiAssistant.chatCompletionError.internalServerError',
+ {
+ defaultMessage: 'An internal server error occurred',
+ }
+ )
+) {
return new ChatCompletionError(ChatCompletionErrorCode.InternalError, originalErrorMessage);
}
diff --git a/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts b/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts
index 2487fca287cc7..8b6ef27ee8ebd 100644
--- a/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts
+++ b/x-pack/plugins/observability_ai_assistant/common/utils/process_openai_stream.ts
@@ -19,7 +19,6 @@ export function processOpenAiStream() {
const id = v4();
return source.pipe(
- map((line) => line.substring(6)),
filter((line) => !!line && line !== '[DONE]'),
map(
(line) =>
diff --git a/x-pack/plugins/observability_ai_assistant/kibana.jsonc b/x-pack/plugins/observability_ai_assistant/kibana.jsonc
index 3f346cccff0c1..a3eaad0d216a3 100644
--- a/x-pack/plugins/observability_ai_assistant/kibana.jsonc
+++ b/x-pack/plugins/observability_ai_assistant/kibana.jsonc
@@ -25,7 +25,9 @@
"ml"
],
"requiredBundles": [ "kibanaReact", "kibanaUtils"],
- "optionalPlugins": [],
+ "optionalPlugins": [
+ "cloud"
+ ],
"extraPublicDirs": []
}
}
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx
index bf514691f7d93..0227ef42e2808 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/welcome_message.tsx
@@ -25,6 +25,7 @@ import { Disclaimer } from './disclaimer';
import { WelcomeMessageConnectors } from './welcome_message_connectors';
import { WelcomeMessageKnowledgeBase } from './welcome_message_knowledge_base';
import { useKibana } from '../../hooks/use_kibana';
+import { isSupportedConnectorType } from '../../../common/connectors';
const fullHeightClassName = css`
height: 100%;
@@ -68,7 +69,7 @@ export function WelcomeMessage({
const onConnectorCreated = (createdConnector: ActionConnector) => {
setConnectorFlyoutOpen(false);
- if (createdConnector.actionTypeId === '.gen-ai') {
+ if (isSupportedConnectorType(createdConnector.actionTypeId)) {
connectors.reloadConnectors();
}
diff --git a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
index 17ea46c56681e..5fe933835eecd 100644
--- a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
@@ -9,8 +9,10 @@ import { AnalyticsServiceStart, HttpResponse } from '@kbn/core/public';
import { AbortError } from '@kbn/kibana-utils-plugin/common';
import { IncomingMessage } from 'http';
import { pick } from 'lodash';
-import { concatMap, delay, map, Observable, of, scan, shareReplay, timestamp } from 'rxjs';
+import { concatMap, delay, filter, map, Observable, of, scan, shareReplay, timestamp } from 'rxjs';
import {
+ BufferFlushEvent,
+ StreamingChatResponseEventType,
StreamingChatResponseEventWithoutError,
type StreamingChatResponseEvent,
} from '../../common/conversation_complete';
@@ -163,7 +165,11 @@ export async function createChatService({
const response = _response as unknown as HttpResponse;
const response$ = toObservable(response)
.pipe(
- map((line) => JSON.parse(line) as StreamingChatResponseEvent),
+ map((line) => JSON.parse(line) as StreamingChatResponseEvent | BufferFlushEvent),
+ filter(
+ (line): line is StreamingChatResponseEvent =>
+ line.type !== StreamingChatResponseEventType.BufferFlush
+ ),
throwSerializedChatCompletionErrors()
)
.subscribe(subscriber);
@@ -224,7 +230,11 @@ export async function createChatService({
const subscription = toObservable(response)
.pipe(
- map((line) => JSON.parse(line) as StreamingChatResponseEvent),
+ map((line) => JSON.parse(line) as StreamingChatResponseEvent | BufferFlushEvent),
+ filter(
+ (line): line is StreamingChatResponseEvent =>
+ line.type !== StreamingChatResponseEventType.BufferFlush
+ ),
throwSerializedChatCompletionErrors()
)
.subscribe(subscriber);
diff --git a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md
index c5ff90ed582f2..76bf8a7fe7df2 100644
--- a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md
+++ b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/README.md
@@ -26,7 +26,7 @@ By default, the tool will look for a Kibana instance running locally (at `http:/
#### Connector
-Use `--connectorId` to specify a `.gen-ai` connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single `.gen-ai` connector is found, it will be used without prompting.
+Use `--connectorId` to specify a `.gen-ai` or `.bedrock` connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single supported connector is found, it will be used without prompting.
#### Persisting conversations
diff --git a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts
index d77e37a2b55a8..d0aa91f7ac53e 100644
--- a/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts
+++ b/x-pack/plugins/observability_ai_assistant/scripts/evaluation/kibana_client.ts
@@ -12,7 +12,9 @@ import { format, parse, UrlObject } from 'url';
import { ToolingLog } from '@kbn/tooling-log';
import pRetry from 'p-retry';
import { Message, MessageRole } from '../../common';
+import { isSupportedConnectorType } from '../../common/connectors';
import {
+ BufferFlushEvent,
ChatCompletionChunkEvent,
ChatCompletionErrorEvent,
ConversationCreateEvent,
@@ -217,7 +219,17 @@ export class KibanaClient {
)
).data
).pipe(
- map((line) => JSON.parse(line) as ChatCompletionChunkEvent | ChatCompletionErrorEvent),
+ map(
+ (line) =>
+ JSON.parse(line) as
+ | ChatCompletionChunkEvent
+ | ChatCompletionErrorEvent
+ | BufferFlushEvent
+ ),
+ filter(
+ (line): line is ChatCompletionChunkEvent | ChatCompletionErrorEvent =>
+ line.type !== StreamingChatResponseEventType.BufferFlush
+ ),
throwSerializedChatCompletionErrors(),
concatenateChatCompletionChunks()
);
@@ -270,13 +282,13 @@ export class KibanaClient {
)
).data
).pipe(
- map((line) => JSON.parse(line) as StreamingChatResponseEvent),
- throwSerializedChatCompletionErrors(),
+ map((line) => JSON.parse(line) as StreamingChatResponseEvent | BufferFlushEvent),
filter(
(event): event is MessageAddEvent | ConversationCreateEvent =>
event.type === StreamingChatResponseEventType.MessageAdd ||
event.type === StreamingChatResponseEventType.ConversationCreate
),
+ throwSerializedChatCompletionErrors(),
toArray()
);
@@ -427,6 +439,8 @@ export class KibanaClient {
})
);
- return connectors.data.filter((connector) => connector.connector_type_id === '.gen-ai');
+ return connectors.data.filter((connector) =>
+ isSupportedConnectorType(connector.connector_type_id)
+ );
}
}
diff --git a/x-pack/plugins/observability_ai_assistant/server/functions/index.ts b/x-pack/plugins/observability_ai_assistant/server/functions/index.ts
index d02f943c3523e..708f77da33321 100644
--- a/x-pack/plugins/observability_ai_assistant/server/functions/index.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/functions/index.ts
@@ -53,7 +53,6 @@ export const registerFunctions: ChatRegistrationFunction = async ({
If multiple functions are suitable, use the most specific and easy one. E.g., when the user asks to visualise APM data, use the APM functions (if available) rather than "query".
- Use the "get_dataset_info" function if it is not clear what fields or indices the user means, or if you want to get more information about the mappings.
Note that ES|QL (the Elasticsearch query language, which is NOT Elasticsearch SQL, but a new piped language) is the preferred query language.
@@ -66,6 +65,8 @@ export const registerFunctions: ChatRegistrationFunction = async ({
When the "visualize_query" function has been called, a visualization has been displayed to the user. DO NOT UNDER ANY CIRCUMSTANCES follow up a "visualize_query" function call with your own visualization attempt.
If the "execute_query" function has been called, summarize these results for the user. The user does not see a visualization in this case.
+ Use the "get_dataset_info" function if it is not clear what fields or indices the user means, or if you want to get more information about the mappings.
+
If the "get_dataset_info" function returns no data, and the user asks for a query, generate a query anyway with the "query" function, but be explicit about it potentially being incorrect.
`
);
diff --git a/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts b/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts
index b69188d81b84a..86da0c0395587 100644
--- a/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/functions/query/index.ts
@@ -113,6 +113,7 @@ export function registerQueryFunction({
type: 'boolean',
},
},
+ required: ['switch'],
} as const,
},
async ({ messages, connectorId }, signal) => {
@@ -129,54 +130,58 @@ export function registerQueryFunction({
const source$ = (
await client.chat('classify_esql', {
connectorId,
- messages: withEsqlSystemMessage(
- `Use the classify_esql function to classify the user's request
- and get more information about specific functions and commands
- you think are candidates for answering the question.
-
+ messages: withEsqlSystemMessage().concat({
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.User,
+ content: `Use the classify_esql function to classify the user's request
+ in the user message before this.
+ and get more information about specific functions and commands
+ you think are candidates for answering the question.
- Examples for functions and commands:
- Do you need to group data? Request \`STATS\`.
- Extract data? Request \`DISSECT\` AND \`GROK\`.
- Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`.
-
- For determining the intention of the user, the following options are available:
-
- ${VisualizeESQLUserIntention.generateQueryOnly}: the user only wants to generate the query,
- but not run it.
-
- ${VisualizeESQLUserIntention.executeAndReturnResults}: the user wants to execute the query,
- and have the assistant return/analyze/summarize the results. they don't need a
- visualization.
-
- ${VisualizeESQLUserIntention.visualizeAuto}: The user wants to visualize the data from the
- query, but wants us to pick the best visualization type, or their preferred
- visualization is unclear.
-
- These intentions will display a specific visualization:
- ${VisualizeESQLUserIntention.visualizeBar}
- ${VisualizeESQLUserIntention.visualizeDonut}
- ${VisualizeESQLUserIntention.visualizeHeatmap}
- ${VisualizeESQLUserIntention.visualizeLine}
- ${VisualizeESQLUserIntention.visualizeTagcloud}
- ${VisualizeESQLUserIntention.visualizeTreemap}
- ${VisualizeESQLUserIntention.visualizeWaffle}
- ${VisualizeESQLUserIntention.visualizeXy}
-
- Some examples:
- "Show me the avg of x" => ${VisualizeESQLUserIntention.executeAndReturnResults}
- "Show me the results of y" => ${VisualizeESQLUserIntention.executeAndReturnResults}
- "Display the sum of z" => ${VisualizeESQLUserIntention.executeAndReturnResults}
-
- "I want a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly}
- "... Just show me the query" => ${VisualizeESQLUserIntention.generateQueryOnly}
- "Create a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly}
-
- "Show me the avg of x over time" => ${VisualizeESQLUserIntention.visualizeAuto}
- "I want a bar chart of ... " => ${VisualizeESQLUserIntention.visualizeBar}
- "I want to see a heat map of ..." => ${VisualizeESQLUserIntention.visualizeHeatmap}
- `
- ),
+ Examples for functions and commands:
+ Do you need to group data? Request \`STATS\`.
+ Extract data? Request \`DISSECT\` AND \`GROK\`.
+ Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`.
+
+ For determining the intention of the user, the following options are available:
+
+ ${VisualizeESQLUserIntention.generateQueryOnly}: the user only wants to generate the query,
+ but not run it.
+
+ ${VisualizeESQLUserIntention.executeAndReturnResults}: the user wants to execute the query,
+ and have the assistant return/analyze/summarize the results. they don't need a
+ visualization.
+
+ ${VisualizeESQLUserIntention.visualizeAuto}: The user wants to visualize the data from the
+ query, but wants us to pick the best visualization type, or their preferred
+ visualization is unclear.
+
+ These intentions will display a specific visualization:
+ ${VisualizeESQLUserIntention.visualizeBar}
+ ${VisualizeESQLUserIntention.visualizeDonut}
+ ${VisualizeESQLUserIntention.visualizeHeatmap}
+ ${VisualizeESQLUserIntention.visualizeLine}
+ ${VisualizeESQLUserIntention.visualizeTagcloud}
+ ${VisualizeESQLUserIntention.visualizeTreemap}
+ ${VisualizeESQLUserIntention.visualizeWaffle}
+ ${VisualizeESQLUserIntention.visualizeXy}
+
+ Some examples:
+ "Show me the avg of x" => ${VisualizeESQLUserIntention.executeAndReturnResults}
+ "Show me the results of y" => ${VisualizeESQLUserIntention.executeAndReturnResults}
+ "Display the sum of z" => ${VisualizeESQLUserIntention.executeAndReturnResults}
+
+ "I want a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly}
+ "... Just show me the query" => ${VisualizeESQLUserIntention.generateQueryOnly}
+ "Create a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly}
+
+ "Show me the avg of x over time" => ${VisualizeESQLUserIntention.visualizeAuto}
+ "I want a bar chart of ... " => ${VisualizeESQLUserIntention.visualizeBar}
+ "I want to see a heat map of ..." => ${VisualizeESQLUserIntention.visualizeHeatmap}
+ `,
+ },
+ }),
signal,
functions: [
{
@@ -184,6 +189,9 @@ export function registerQueryFunction({
description: `Use this function to determine:
- what ES|QL functions and commands are candidates for answering the user's question
- whether the user has requested a query, and if so, it they want it to be executed, or just shown.
+
+ All parameters are required. Make sure the functions and commands you request are available in the
+ system message.
`,
parameters: {
type: 'object',
@@ -218,6 +226,10 @@ export function registerQueryFunction({
const response = await lastValueFrom(source$);
+ if (!response.message.function_call.arguments) {
+ throw new Error('LLM did not call classify_esql function');
+ }
+
const args = JSON.parse(response.message.function_call.arguments) as {
commands: string[];
functions: string[];
diff --git a/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts b/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts
index 909a823286cc6..125c9a2f6eea0 100644
--- a/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/functions/recall.ts
@@ -11,11 +11,11 @@ import dedent from 'dedent';
import * as t from 'io-ts';
import { compact, last, omit } from 'lodash';
import { lastValueFrom } from 'rxjs';
+import { Logger } from '@kbn/logging';
import { FunctionRegistrationParameters } from '.';
import { MessageRole, type Message } from '../../common/types';
import { concatenateChatCompletionChunks } from '../../common/utils/concatenate_chat_completion_chunks';
import type { ObservabilityAIAssistantClient } from '../service/client';
-import { RespondFunctionResources } from '../service/types';
export function registerRecallFunction({
client,
@@ -114,7 +114,7 @@ export function registerRecallFunction({
client,
connectorId,
signal,
- resources,
+ logger: resources.logger,
});
return {
@@ -162,7 +162,7 @@ async function scoreSuggestions({
client,
connectorId,
signal,
- resources,
+ logger,
}: {
suggestions: Awaited>;
messages: Message[];
@@ -170,7 +170,7 @@ async function scoreSuggestions({
client: ObservabilityAIAssistantClient;
connectorId: string;
signal: AbortSignal;
- resources: RespondFunctionResources;
+ logger: Logger;
}) {
const indexedSuggestions = suggestions.map((suggestion, index) => ({ ...suggestion, id: index }));
@@ -233,6 +233,7 @@ async function scoreSuggestions({
})
).pipe(concatenateChatCompletionChunks())
);
+
const scoreFunctionRequest = decodeOrThrow(scoreFunctionRequestRt)(response);
const { scores: scoresAsString } = decodeOrThrow(jsonRt.pipe(scoreFunctionArgumentsRt))(
scoreFunctionRequest.message.function_call.arguments
@@ -264,10 +265,7 @@ async function scoreSuggestions({
relevantDocumentIds.includes(suggestion.id)
);
- resources.logger.debug(
- `Found ${relevantDocumentIds.length} relevant suggestions from the knowledge base. ${scores.length} suggestions were considered in total.`
- );
- resources.logger.debug(`Relevant documents: ${JSON.stringify(relevantDocuments, null, 2)}`);
+ logger.debug(`Relevant documents: ${JSON.stringify(relevantDocuments, null, 2)}`);
return relevantDocuments;
}
diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts
index 517cc48f9f27c..a9c58a9a59e00 100644
--- a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts
@@ -5,13 +5,13 @@
* 2.0.
*/
import { notImplemented } from '@hapi/boom';
-import * as t from 'io-ts';
import { toBooleanRt } from '@kbn/io-ts-utils';
-import type OpenAI from 'openai';
+import * as t from 'io-ts';
import { Readable } from 'stream';
+import { flushBuffer } from '../../service/util/flush_buffer';
+import { observableIntoStream } from '../../service/util/observable_into_stream';
import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route';
import { messageRt } from '../runtime_types';
-import { observableIntoStream } from '../../service/util/observable_into_stream';
const chatRoute = createObservabilityAIAssistantServerRoute({
endpoint: 'POST /internal/observability_ai_assistant/chat',
@@ -40,7 +40,10 @@ const chatRoute = createObservabilityAIAssistantServerRoute({
handler: async (resources): Promise => {
const { request, params, service } = resources;
- const client = await service.getClient({ request });
+ const [client, cloudStart] = await Promise.all([
+ service.getClient({ request }),
+ resources.plugins.cloud?.start(),
+ ]);
if (!client) {
throw notImplemented();
@@ -68,7 +71,7 @@ const chatRoute = createObservabilityAIAssistantServerRoute({
: {}),
});
- return observableIntoStream(response$);
+ return observableIntoStream(response$.pipe(flushBuffer(!!cloudStart?.isCloudEnabled)));
},
});
@@ -90,10 +93,13 @@ const chatCompleteRoute = createObservabilityAIAssistantServerRoute({
}),
]),
}),
- handler: async (resources): Promise => {
+ handler: async (resources): Promise => {
const { request, params, service } = resources;
- const client = await service.getClient({ request });
+ const [client, cloudStart] = await Promise.all([
+ service.getClient({ request }),
+ resources.plugins.cloud?.start() || Promise.resolve(undefined),
+ ]);
if (!client) {
throw notImplemented();
@@ -125,7 +131,7 @@ const chatCompleteRoute = createObservabilityAIAssistantServerRoute({
functionClient,
});
- return observableIntoStream(response$);
+ return observableIntoStream(response$.pipe(flushBuffer(!!cloudStart?.isCloudEnabled)));
},
});
diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts
index 894896fec6b3c..79134b9fef8d0 100644
--- a/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/routes/connectors/route.ts
@@ -5,6 +5,7 @@
* 2.0.
*/
import { FindActionResult } from '@kbn/actions-plugin/server';
+import { isSupportedConnectorType } from '../../../common/connectors';
import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route';
const listConnectorsRoute = createObservabilityAIAssistantServerRoute({
@@ -21,7 +22,7 @@ const listConnectorsRoute = createObservabilityAIAssistantServerRoute({
const connectors = await actionsClient.getAll();
- return connectors.filter((connector) => connector.actionTypeId === '.gen-ai');
+ return connectors.filter((connector) => isSupportedConnectorType(connector.actionTypeId));
},
});
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.test.ts
new file mode 100644
index 0000000000000..e92d14088d337
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.test.ts
@@ -0,0 +1,239 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { Logger } from '@kbn/logging';
+import dedent from 'dedent';
+import { last } from 'lodash';
+import { MessageRole } from '../../../../common';
+import { createBedrockClaudeAdapter } from './bedrock_claude_adapter';
+import { LlmApiAdapterFactory } from './types';
+
+describe('createBedrockClaudeAdapter', () => {
+ describe('getSubAction', () => {
+ function callSubActionFactory(overrides?: Partial[0]>) {
+ const subActionParams = createBedrockClaudeAdapter({
+ logger: {
+ debug: jest.fn(),
+ } as unknown as Logger,
+ functions: [
+ {
+ name: 'my_tool',
+ description: 'My tool',
+ parameters: {
+ properties: {
+ myParam: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ ],
+ messages: [
+ {
+ '@timestamp': new Date().toString(),
+ message: {
+ role: MessageRole.System,
+ content: '',
+ },
+ },
+ {
+ '@timestamp': new Date().toString(),
+ message: {
+ role: MessageRole.User,
+ content: 'How can you help me?',
+ },
+ },
+ ],
+ ...overrides,
+ }).getSubAction().subActionParams as {
+ temperature: number;
+ messages: Array<{ role: string; content: string }>;
+ };
+
+ return {
+ ...subActionParams,
+ messages: subActionParams.messages.map((msg) => ({ ...msg, content: dedent(msg.content) })),
+ };
+ }
+ describe('with functions', () => {
+ it('sets the temperature to 0', () => {
+ expect(callSubActionFactory().temperature).toEqual(0);
+ });
+
+ it('formats the functions', () => {
+ expect(callSubActionFactory().messages[0].content).toContain(
+ dedent(`
+
+ my_tool
+ My tool
+
+
+ myParam
+ string
+
+
+ Required: false
+ Multiple: false
+
+
+
+
+
+ `)
+ );
+ });
+
+ it('replaces mentions of functions with tools', () => {
+ const messages = [
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.System,
+ content:
+ 'Call the "esql" tool. You can chain successive function calls, using the functions available.',
+ },
+ },
+ ];
+
+ const content = callSubActionFactory({ messages }).messages[0].content;
+
+ expect(content).not.toContain(`"esql" function`);
+ expect(content).toContain(`"esql" tool`);
+ expect(content).not.toContain(`functions`);
+ expect(content).toContain(`tools`);
+ expect(content).toContain(`function calls`);
+ });
+
+ it('mentions to explicitly call the specified function if given', () => {
+ expect(last(callSubActionFactory({ functionCall: 'my_tool' }).messages)!.content).toContain(
+ 'Remember, use the my_tool tool to answer this question.'
+ );
+ });
+
+ it('formats the function requests as XML', () => {
+ const messages = [
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.System,
+ content: '',
+ },
+ },
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.Assistant,
+ function_call: {
+ name: 'my_tool',
+ arguments: JSON.stringify({ myParam: 'myValue' }),
+ trigger: MessageRole.User as const,
+ },
+ },
+ },
+ ];
+
+ expect(last(callSubActionFactory({ messages }).messages)!.content).toContain(
+ dedent(`
+
+ my_tool
+
+ myValue
+
+
+ `)
+ );
+ });
+
+ it('formats errors', () => {
+ const messages = [
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.System,
+ content: '',
+ },
+ },
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.Assistant,
+ function_call: {
+ name: 'my_tool',
+ arguments: JSON.stringify({ myParam: 'myValue' }),
+ trigger: MessageRole.User as const,
+ },
+ },
+ },
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.User,
+ name: 'my_tool',
+ content: JSON.stringify({ error: 'An internal server error occurred' }),
+ },
+ },
+ ];
+
+ expect(last(callSubActionFactory({ messages }).messages)!.content).toContain(
+ dedent(`
+
+ An internal server error occurred
+
+ `)
+ );
+ });
+
+ it('formats function responses as XML + JSON', () => {
+ const messages = [
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.System,
+ content: '',
+ },
+ },
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.Assistant,
+ function_call: {
+ name: 'my_tool',
+ arguments: JSON.stringify({ myParam: 'myValue' }),
+ trigger: MessageRole.User as const,
+ },
+ },
+ },
+ {
+ '@timestamp': new Date().toISOString(),
+ message: {
+ role: MessageRole.User,
+ name: 'my_tool',
+ content: JSON.stringify({ myResponse: { myParam: 'myValue' } }),
+ },
+ },
+ ];
+
+ expect(last(callSubActionFactory({ messages }).messages)!.content).toContain(
+ dedent(`
+
+ my_tool
+
+
+myValue
+
+
+
+ `)
+ );
+ });
+ });
+ });
+
+ describe('streamIntoObservable', () => {
+ // this data format is heavily encoded, so hard to reproduce.
+ // will leave this empty until we have some sample data.
+ });
+});
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.ts
new file mode 100644
index 0000000000000..d5ba0d726ab12
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/bedrock_claude_adapter.ts
@@ -0,0 +1,228 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import dedent from 'dedent';
+import { castArray } from 'lodash';
+import { filter, tap } from 'rxjs';
+import { Builder } from 'xml2js';
+import { createInternalServerError } from '../../../../common/conversation_complete';
+import {
+ BedrockChunkMember,
+ eventstreamSerdeIntoObservable,
+} from '../../util/eventstream_serde_into_observable';
+import { jsonSchemaToFlatParameters } from '../../util/json_schema_to_flat_parameters';
+import { processBedrockStream } from './process_bedrock_stream';
+import type { LlmApiAdapterFactory } from './types';
+
+function replaceFunctionsWithTools(content: string) {
+ return content.replaceAll(/(function)(s)?(?!\scall)/g, (match, p1, p2) => {
+ return `tool${p2 || ''}`;
+ });
+}
+
+// Most of the work here is to re-format OpenAI-compatible functions for Claude.
+// See https://github.com/anthropics/anthropic-tools/blob/main/tool_use_package/prompt_constructors.py
+
+export const createBedrockClaudeAdapter: LlmApiAdapterFactory = ({
+ messages,
+ functions,
+ functionCall,
+ logger,
+}) => ({
+ getSubAction: () => {
+ const [systemMessage, ...otherMessages] = messages;
+
+ const filteredFunctions = functionCall
+ ? functions?.filter((fn) => fn.name === functionCall)
+ : functions;
+
+ let functionsPrompt: string = '';
+
+ if (filteredFunctions?.length) {
+ functionsPrompt = `In this environment, you have access to a set of tools you can use to answer the user's question.
+
+ When deciding what tool to use, keep in mind that you can call other tools in successive requests, so decide what tool
+ would be a good first step.
+
+ You MUST only invoke a single tool, and invoke it once. Other invocations will be ignored.
+ You MUST wait for the results before invoking another.
+ You can call multiple tools in successive messages. This means you can chain function calls. If any tool was used in a previous
+ message, consider whether it still makes sense to follow it up with another function call.
+
+ ${
+ functions?.find((fn) => fn.name === 'recall')
+ ? `The "recall" function is ALWAYS used after a user question. Even if it was used before, your job is to answer the last user question,
+ even if the "recall" function was executed after that. Consider the tools you need to answer the user's question.`
+ : ''
+ }
+
+ Rather than explaining how you would call a function, just generate the XML to call the function. It will automatically be
+ executed and returned to you.
+
+ These results are generally not visible to the user. Treat them as if they are not,
+ unless specified otherwise.
+
+ ONLY respond with XML, do not add any text.
+
+ If a parameter allows multiple values, separate the values by ","
+
+ You may call them like this.
+
+
+
+ $TOOL_NAME
+
+ <$PARAMETER_NAME>$PARAMETER_VALUE
+ ...
+
+
+
+
+ Here are the tools available:
+
+
+ ${filteredFunctions
+ .map(
+ (fn) => `
+ ${fn.name}
+ ${fn.description}
+
+ ${jsonSchemaToFlatParameters(fn.parameters).map((param) => {
+ return `
+ ${param.name}
+ ${param.type}
+
+ ${param.description || ''}
+ Required: ${!!param.required}
+ Multiple: ${!!param.array}
+ ${
+ param.enum || param.constant
+ ? `Allowed values: ${castArray(param.constant || param.enum).join(', ')}`
+ : ''
+ }
+
+ `;
+ })}
+
+ `
+ )
+ .join('\n')}
+
+
+
+ Examples:
+
+ Assistant:
+
+
+ my_tool
+
+ foo
+
+
+
+
+ Assistant:
+
+
+ another_tool
+
+ foo
+
+
+
+
+ `;
+ }
+
+ const formattedMessages = [
+ {
+ role: 'system',
+ content: `${replaceFunctionsWithTools(systemMessage.message.content!)}
+
+ ${functionsPrompt}
+ `,
+ },
+ ...otherMessages.map((message, index) => {
+ const builder = new Builder({ headless: true });
+ if (message.message.name) {
+ const deserialized = JSON.parse(message.message.content || '{}');
+
+ if ('error' in deserialized) {
+ return {
+ role: message.message.role,
+ content: dedent(`
+
+ ${builder.buildObject(deserialized)}
+
+
+ `),
+ };
+ }
+
+ return {
+ role: message.message.role,
+ content: dedent(`
+
+
+ ${message.message.name}
+
+ ${builder.buildObject(deserialized)}
+
+
+ `),
+ };
+ }
+
+ let content = replaceFunctionsWithTools(message.message.content || '');
+
+ if (message.message.function_call?.name) {
+ content += builder.buildObject({
+ function_calls: {
+ invoke: {
+ tool_name: message.message.function_call.name,
+ parameters: JSON.parse(message.message.function_call.arguments || '{}'),
+ },
+ },
+ });
+ }
+
+ if (index === otherMessages.length - 1 && functionCall) {
+ content += `
+
+ Remember, use the ${functionCall} tool to answer this question.`;
+ }
+
+ return {
+ role: message.message.role,
+ content,
+ };
+ }),
+ ];
+
+ return {
+ subAction: 'invokeStream',
+ subActionParams: {
+ messages: formattedMessages,
+ temperature: 0,
+ stopSequences: ['\n\nHuman:', ''],
+ },
+ };
+ },
+ streamIntoObservable: (readable) =>
+ eventstreamSerdeIntoObservable(readable).pipe(
+ tap((value) => {
+ if ('modelStreamErrorException' in value) {
+ throw createInternalServerError(value.modelStreamErrorException.originalMessage);
+ }
+ }),
+ filter((value): value is BedrockChunkMember => {
+ return 'chunk' in value && value.chunk?.headers?.[':event-type']?.value === 'chunk';
+ }),
+ processBedrockStream({ logger, functions })
+ ),
+});
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/openai_adapter.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/openai_adapter.ts
new file mode 100644
index 0000000000000..61935d891a1db
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/openai_adapter.ts
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { compact, isEmpty, omit } from 'lodash';
+import OpenAI from 'openai';
+import { MessageRole } from '../../../../common';
+import { processOpenAiStream } from '../../../../common/utils/process_openai_stream';
+import { eventsourceStreamIntoObservable } from '../../util/eventsource_stream_into_observable';
+import { LlmApiAdapterFactory } from './types';
+
+export const createOpenAiAdapter: LlmApiAdapterFactory = ({
+ messages,
+ functions,
+ functionCall,
+ logger,
+}) => {
+ return {
+ getSubAction: () => {
+ const messagesForOpenAI: Array<
+ Omit & {
+ role: MessageRole;
+ }
+ > = compact(
+ messages
+ .filter((message) => message.message.content || message.message.function_call?.name)
+ .map((message) => {
+ const role =
+ message.message.role === MessageRole.Elastic
+ ? MessageRole.User
+ : message.message.role;
+
+ return {
+ role,
+ content: message.message.content,
+ function_call: isEmpty(message.message.function_call?.name)
+ ? undefined
+ : omit(message.message.function_call, 'trigger'),
+ name: message.message.name,
+ };
+ })
+ );
+
+ const functionsForOpenAI = functions;
+
+ const request: Omit & { model?: string } = {
+ messages: messagesForOpenAI as OpenAI.ChatCompletionCreateParams['messages'],
+ stream: true,
+ ...(!!functions?.length ? { functions: functionsForOpenAI } : {}),
+ temperature: 0,
+ function_call: functionCall ? { name: functionCall } : undefined,
+ };
+
+ return {
+ subAction: 'stream',
+ subActionParams: {
+ body: JSON.stringify(request),
+ stream: true,
+ },
+ };
+ },
+ streamIntoObservable: (readable) => {
+ return eventsourceStreamIntoObservable(readable).pipe(processOpenAiStream());
+ },
+ };
+};
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.test.ts
new file mode 100644
index 0000000000000..78775b4d79d51
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.test.ts
@@ -0,0 +1,256 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { fromUtf8 } from '@smithy/util-utf8';
+import { lastValueFrom, of } from 'rxjs';
+import { Logger } from '@kbn/logging';
+import { concatenateChatCompletionChunks } from '../../../../common/utils/concatenate_chat_completion_chunks';
+import { processBedrockStream } from './process_bedrock_stream';
+import { MessageRole } from '../../../../common';
+
+describe('processBedrockStream', () => {
+ const encode = (completion: string, stop?: string) => {
+ return {
+ chunk: {
+ headers: {
+ '::event-type': { value: 'chunk', type: 'uuid' as const },
+ },
+ body: fromUtf8(
+ JSON.stringify({
+ bytes: Buffer.from(JSON.stringify({ completion, stop }), 'utf-8').toString('base64'),
+ })
+ ),
+ },
+ };
+ };
+
+ function getLoggerMock() {
+ return {
+ debug: jest.fn(),
+ } as unknown as Logger;
+ }
+
+ it('parses normal text messages', async () => {
+ expect(
+ await lastValueFrom(
+ of(encode('This'), encode(' is'), encode(' some normal'), encode(' text')).pipe(
+ processBedrockStream({ logger: getLoggerMock() }),
+ concatenateChatCompletionChunks()
+ )
+ )
+ ).toEqual({
+ message: {
+ content: 'This is some normal text',
+ function_call: {
+ arguments: '',
+ name: '',
+ trigger: MessageRole.Assistant,
+ },
+ role: MessageRole.Assistant,
+ },
+ });
+ });
+
+ it('parses function calls when no text is given', async () => {
+ expect(
+ await lastValueFrom(
+ of(
+ encode('my_toolmy_value', '')
+ ).pipe(
+ processBedrockStream({
+ logger: getLoggerMock(),
+ functions: [
+ {
+ name: 'my_tool',
+ description: '',
+ parameters: {
+ properties: {
+ my_param: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ ],
+ }),
+ concatenateChatCompletionChunks()
+ )
+ )
+ ).toEqual({
+ message: {
+ content: '',
+ function_call: {
+ arguments: JSON.stringify({ my_param: 'my_value' }),
+ name: 'my_tool',
+ trigger: MessageRole.Assistant,
+ },
+ role: MessageRole.Assistant,
+ },
+ });
+ });
+
+ it('parses function calls when they are prefaced by text', async () => {
+ expect(
+ await lastValueFrom(
+ of(
+ encode('This is'),
+ encode(' my text\nmy_toolmy_value', '')
+ ).pipe(
+ processBedrockStream({
+ logger: getLoggerMock(),
+ functions: [
+ {
+ name: 'my_tool',
+ description: '',
+ parameters: {
+ properties: {
+ my_param: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ ],
+ }),
+ concatenateChatCompletionChunks()
+ )
+ )
+ ).toEqual({
+ message: {
+ content: 'This is my text',
+ function_call: {
+ arguments: JSON.stringify({ my_param: 'my_value' }),
+ name: 'my_tool',
+ trigger: MessageRole.Assistant,
+ },
+ role: MessageRole.Assistant,
+ },
+ });
+ });
+
+ it('throws an error if the XML cannot be parsed', async () => {
+ expect(
+ async () =>
+ await lastValueFrom(
+ of(
+ encode('my_toolmy_value', '')
+ ).pipe(
+ processBedrockStream({
+ logger: getLoggerMock(),
+ functions: [
+ {
+ name: 'my_tool',
+ description: '',
+ parameters: {
+ properties: {
+ my_param: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ ],
+ }),
+ concatenateChatCompletionChunks()
+ )
+ )
+ ).rejects.toThrowErrorMatchingInlineSnapshot(`
+ "Unexpected close tag
+ Line: 0
+ Column: 49
+ Char: >"
+ `);
+ });
+
+ it('throws an error if the function does not exist', async () => {
+ expect(
+ async () =>
+ await lastValueFrom(
+ of(
+ encode('my_other_toolmy_value', '')
+ ).pipe(
+ processBedrockStream({
+ logger: getLoggerMock(),
+ functions: [
+ {
+ name: 'my_tool',
+ description: '',
+ parameters: {
+ properties: {
+ my_param: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ ],
+ }),
+ concatenateChatCompletionChunks()
+ )
+ )
+ ).rejects.toThrowError(
+ 'Function definition for my_other_tool not found. Available are: my_tool'
+ );
+ });
+
+ it('successfully invokes a function without parameters', async () => {
+ expect(
+ await lastValueFrom(
+ of(
+ encode('my_tool', '')
+ ).pipe(
+ processBedrockStream({
+ logger: getLoggerMock(),
+ functions: [
+ {
+ name: 'my_tool',
+ description: '',
+ parameters: {
+ properties: {
+ my_param: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ ],
+ }),
+ concatenateChatCompletionChunks()
+ )
+ )
+ ).toEqual({
+ message: {
+ content: '',
+ function_call: {
+ arguments: '{}',
+ name: 'my_tool',
+ trigger: MessageRole.Assistant,
+ },
+ role: MessageRole.Assistant,
+ },
+ });
+ });
+});
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.ts
new file mode 100644
index 0000000000000..41bc19717485c
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/process_bedrock_stream.ts
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { toUtf8 } from '@smithy/util-utf8';
+import { Observable } from 'rxjs';
+import { v4 } from 'uuid';
+import { Parser } from 'xml2js';
+import type { Logger } from '@kbn/logging';
+import { JSONSchema } from 'json-schema-to-ts';
+import {
+ ChatCompletionChunkEvent,
+ createInternalServerError,
+ StreamingChatResponseEventType,
+} from '../../../../common/conversation_complete';
+import type { BedrockChunkMember } from '../../util/eventstream_serde_into_observable';
+import { convertDeserializedXmlWithJsonSchema } from '../../util/convert_deserialized_xml_with_json_schema';
+
+async function parseFunctionCallXml({
+ xml,
+ functions,
+}: {
+ xml: string;
+ functions?: Array<{ name: string; description: string; parameters: JSONSchema }>;
+}) {
+ const parser = new Parser();
+
+ const parsedValue = await parser.parseStringPromise(xml);
+ const invoke = parsedValue.function_calls.invoke[0];
+ const fnName = invoke.tool_name[0];
+ const parameters: Array> = invoke.parameters ?? [];
+ const functionDef = functions?.find((fn) => fn.name === fnName);
+
+ if (!functionDef) {
+ throw createInternalServerError(
+ `Function definition for ${fnName} not found. ${
+ functions?.length
+ ? 'Available are: ' + functions.map((fn) => fn.name).join(', ') + '.'
+ : 'No functions are available.'
+ }`
+ );
+ }
+
+ const args = convertDeserializedXmlWithJsonSchema(parameters, functionDef.parameters);
+
+ return {
+ name: fnName,
+ arguments: JSON.stringify(args),
+ };
+}
+
+export function processBedrockStream({
+ logger,
+ functions,
+}: {
+ logger: Logger;
+ functions?: Array<{ name: string; description: string; parameters: JSONSchema }>;
+}) {
+ return (source: Observable) =>
+ new Observable((subscriber) => {
+ let functionCallsBuffer: string = '';
+ const id = v4();
+
+ // We use this to make sure we don't complete the Observable
+ // before all operations have completed.
+ let nextPromise = Promise.resolve();
+
+ // As soon as we see a `';
+
+ const isInFunctionCall = !!functionCallsBuffer;
+
+ if (isStartOfFunctionCall) {
+ const [before, after] = completion.split(' {
+ subscriber.next({
+ id,
+ type: StreamingChatResponseEventType.ChatCompletionChunk,
+ message: {
+ content: index === parts.length - 1 ? part : part + ' ',
+ },
+ });
+ });
+ }
+ }
+
+ source.subscribe({
+ next: (value) => {
+ nextPromise = nextPromise.then(() =>
+ handleNext(value).catch((error) => subscriber.error(error))
+ );
+ },
+ error: (err) => {
+ subscriber.error(err);
+ },
+ complete: () => {
+ nextPromise.then(() => subscriber.complete());
+ },
+ });
+ });
+}
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/types.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/types.ts
new file mode 100644
index 0000000000000..6ef3611bb4aae
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/adapters/types.ts
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Readable } from 'node:stream';
+import type { Observable } from 'rxjs';
+import type { Logger } from '@kbn/logging';
+import type { Message } from '../../../../common';
+import type { ChatCompletionChunkEvent } from '../../../../common/conversation_complete';
+import type { CompatibleJSONSchema } from '../../../../common/types';
+
+export type LlmApiAdapterFactory = (options: {
+ logger: Logger;
+ messages: Message[];
+ functions?: Array<{ name: string; description: string; parameters: CompatibleJSONSchema }>;
+ functionCall?: string;
+}) => LlmApiAdapter;
+
+export interface LlmApiAdapter {
+ getSubAction: () => { subAction: string; subActionParams: Record };
+ streamIntoObservable: (readable: Readable) => Observable;
+}
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts
index fb22828247474..cbcbf0ea3fa3a 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/index.test.ts
@@ -16,6 +16,7 @@ import { finished } from 'stream/promises';
import { ObservabilityAIAssistantClient } from '.';
import { createResourceNamesMap } from '..';
import { MessageRole, type Message } from '../../../common';
+import { ObservabilityAIAssistantConnectorType } from '../../../common/connectors';
import {
ChatCompletionChunkEvent,
ChatCompletionErrorCode,
@@ -63,7 +64,7 @@ function createLlmSimulator() {
],
};
await new Promise((resolve, reject) => {
- stream.write(`data: ${JSON.stringify(chunk)}\n`, undefined, (err) => {
+ stream.write(`data: ${JSON.stringify(chunk)}\n\n`, undefined, (err) => {
return err ? reject(err) : resolve();
});
});
@@ -72,7 +73,7 @@ function createLlmSimulator() {
if (stream.destroyed) {
throw new Error('Stream is already destroyed');
}
- await new Promise((resolve) => stream.write('data: [DONE]', () => stream.end(resolve)));
+ await new Promise((resolve) => stream.write('data: [DONE]\n\n', () => stream.end(resolve)));
},
error: (error: Error) => {
stream.destroy(error);
@@ -85,6 +86,7 @@ describe('Observability AI Assistant client', () => {
const actionsClientMock: DeeplyMockedKeys = {
execute: jest.fn(),
+ get: jest.fn(),
} as any;
const internalUserEsClientMock: DeeplyMockedKeys = {
@@ -125,6 +127,15 @@ describe('Observability AI Assistant client', () => {
return name !== 'recall';
});
+ actionsClientMock.get.mockResolvedValue({
+ actionTypeId: ObservabilityAIAssistantConnectorType.OpenAI,
+ id: 'foo',
+ name: 'My connector',
+ isPreconfigured: false,
+ isDeprecated: false,
+ isSystemAction: false,
+ });
+
currentUserEsClientMock.search.mockResolvedValue({
hits: {
hits: [],
@@ -491,6 +502,8 @@ describe('Observability AI Assistant client', () => {
stream.on('data', dataHandler);
+ await nextTick();
+
await llmSimulator.next({ content: 'Hello' });
await llmSimulator.complete();
@@ -590,6 +603,8 @@ describe('Observability AI Assistant client', () => {
stream.on('data', dataHandler);
+ await nextTick();
+
await llmSimulator.next({ content: 'Hello' });
await new Promise((resolve) =>
@@ -598,7 +613,7 @@ describe('Observability AI Assistant client', () => {
error: {
message: 'Connection unexpectedly closed',
},
- })}\n`,
+ })}\n\n`,
resolve
)
);
@@ -694,6 +709,8 @@ describe('Observability AI Assistant client', () => {
stream.on('data', dataHandler);
+ await nextTick();
+
await llmSimulator.next({
content: 'Hello',
function_call: { name: 'my-function', arguments: JSON.stringify({ foo: 'bar' }) },
@@ -1259,6 +1276,8 @@ describe('Observability AI Assistant client', () => {
await nextLlmCallPromise;
}
+ await nextTick();
+
await requestAlertsFunctionCall();
await requestAlertsFunctionCall();
@@ -1348,6 +1367,8 @@ describe('Observability AI Assistant client', () => {
stream.on('data', dataHandler);
+ await nextTick();
+
await llmSimulator.next({ function_call: { name: 'get_top_alerts' } });
await llmSimulator.complete();
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts
index afd34aa8ea966..2fc6bb7be34cc 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts
@@ -12,12 +12,11 @@ import type { Logger } from '@kbn/logging';
import type { PublicMethodsOf } from '@kbn/utility-types';
import apm from 'elastic-apm-node';
import { decode, encode } from 'gpt-tokenizer';
-import { compact, isEmpty, last, merge, noop, omit, pick, take } from 'lodash';
-import type OpenAI from 'openai';
+import { last, merge, noop, omit, pick, take } from 'lodash';
import {
filter,
- firstValueFrom,
isObservable,
+ last as lastOperator,
lastValueFrom,
Observable,
shareReplay,
@@ -25,13 +24,14 @@ import {
} from 'rxjs';
import { Readable } from 'stream';
import { v4 } from 'uuid';
+import { ObservabilityAIAssistantConnectorType } from '../../../common/connectors';
import {
ChatCompletionChunkEvent,
ChatCompletionErrorEvent,
createConversationNotFoundError,
+ createTokenLimitReachedError,
MessageAddEvent,
StreamingChatResponseEventType,
- createTokenLimitReachedError,
type StreamingChatResponseEvent,
} from '../../../common/conversation_complete';
import {
@@ -47,7 +47,6 @@ import {
} from '../../../common/types';
import { concatenateChatCompletionChunks } from '../../../common/utils/concatenate_chat_completion_chunks';
import { emitWithConcatenatedMessage } from '../../../common/utils/emit_with_concatenated_message';
-import { processOpenAiStream } from '../../../common/utils/process_openai_stream';
import type { ChatFunctionClient } from '../chat_function_client';
import {
KnowledgeBaseEntryOperationType,
@@ -56,7 +55,9 @@ import {
} from '../knowledge_base_service';
import type { ObservabilityAIAssistantResourceNames } from '../types';
import { getAccessQuery } from '../util/get_access_query';
-import { streamIntoObservable } from '../util/stream_into_observable';
+import { createBedrockClaudeAdapter } from './adapters/bedrock_claude_adapter';
+import { createOpenAiAdapter } from './adapters/openai_adapter';
+import { LlmApiAdapter } from './adapters/types';
export class ObservabilityAIAssistantClient {
constructor(
@@ -465,111 +466,102 @@ export class ObservabilityAIAssistantClient {
const spanId = (span?.ids['span.id'] || '').substring(0, 6);
- const messagesForOpenAI: Array<
- Omit & {
- role: MessageRole;
- }
- > = compact(
- messages
- .filter((message) => message.message.content || message.message.function_call?.name)
- .map((message) => {
- const role =
- message.message.role === MessageRole.Elastic ? MessageRole.User : message.message.role;
-
- return {
- role,
- content: message.message.content,
- function_call: isEmpty(message.message.function_call?.name)
- ? undefined
- : omit(message.message.function_call, 'trigger'),
- name: message.message.name,
- };
- })
- );
+ try {
+ const connector = await this.dependencies.actionsClient.get({
+ id: connectorId,
+ });
- const functionsForOpenAI = functions;
+ let adapter: LlmApiAdapter;
+
+ switch (connector.actionTypeId) {
+ case ObservabilityAIAssistantConnectorType.OpenAI:
+ adapter = createOpenAiAdapter({
+ logger: this.dependencies.logger,
+ messages,
+ functionCall,
+ functions,
+ });
+ break;
+
+ case ObservabilityAIAssistantConnectorType.Bedrock:
+ adapter = createBedrockClaudeAdapter({
+ logger: this.dependencies.logger,
+ messages,
+ functionCall,
+ functions,
+ });
+ break;
+
+ default:
+ throw new Error(`Connector type is not supported: ${connector.actionTypeId}`);
+ }
- const request: Omit & { model?: string } = {
- messages: messagesForOpenAI as OpenAI.ChatCompletionCreateParams['messages'],
- stream: true,
- ...(!!functions?.length ? { functions: functionsForOpenAI } : {}),
- temperature: 0,
- function_call: functionCall ? { name: functionCall } : undefined,
- };
+ const subAction = adapter.getSubAction();
- this.dependencies.logger.debug(`Sending conversation to connector`);
- this.dependencies.logger.trace(JSON.stringify(request, null, 2));
+ this.dependencies.logger.debug(`Sending conversation to connector`);
+ this.dependencies.logger.trace(JSON.stringify(subAction.subActionParams, null, 2));
- const now = performance.now();
+ const now = performance.now();
- const executeResult = await this.dependencies.actionsClient.execute({
- actionId: connectorId,
- params: {
- subAction: 'stream',
- subActionParams: {
- body: JSON.stringify(request),
- stream: true,
- },
- },
- });
+ const executeResult = await this.dependencies.actionsClient.execute({
+ actionId: connectorId,
+ params: subAction,
+ });
- this.dependencies.logger.debug(
- `Received action client response: ${executeResult.status} (took: ${Math.round(
- performance.now() - now
- )}ms)${spanId ? ` (${spanId})` : ''}`
- );
+ this.dependencies.logger.debug(
+ `Received action client response: ${executeResult.status} (took: ${Math.round(
+ performance.now() - now
+ )}ms)${spanId ? ` (${spanId})` : ''}`
+ );
- if (executeResult.status === 'error' && executeResult?.serviceMessage) {
- const tokenLimitRegex =
- /This model's maximum context length is (\d+) tokens\. However, your messages resulted in (\d+) tokens/g;
- const tokenLimitRegexResult = tokenLimitRegex.exec(executeResult.serviceMessage);
+ if (executeResult.status === 'error' && executeResult?.serviceMessage) {
+ const tokenLimitRegex =
+ /This model's maximum context length is (\d+) tokens\. However, your messages resulted in (\d+) tokens/g;
+ const tokenLimitRegexResult = tokenLimitRegex.exec(executeResult.serviceMessage);
- if (tokenLimitRegexResult) {
- const [, tokenLimit, tokenCount] = tokenLimitRegexResult;
- throw createTokenLimitReachedError(parseInt(tokenLimit, 10), parseInt(tokenCount, 10));
+ if (tokenLimitRegexResult) {
+ const [, tokenLimit, tokenCount] = tokenLimitRegexResult;
+ throw createTokenLimitReachedError(parseInt(tokenLimit, 10), parseInt(tokenCount, 10));
+ }
}
- }
- if (executeResult.status === 'error') {
- throw internal(`${executeResult?.message} - ${executeResult?.serviceMessage}`);
- }
+ if (executeResult.status === 'error') {
+ throw internal(`${executeResult?.message} - ${executeResult?.serviceMessage}`);
+ }
- const response = executeResult.data as Readable;
+ const response = executeResult.data as Readable;
- signal.addEventListener('abort', () => response.destroy());
+ signal.addEventListener('abort', () => response.destroy());
- const observable = streamIntoObservable(response).pipe(processOpenAiStream(), shareReplay());
+ const response$ = adapter.streamIntoObservable(response).pipe(shareReplay());
- firstValueFrom(observable)
- .catch(noop)
- .finally(() => {
- this.dependencies.logger.debug(
- `Received first value after ${Math.round(performance.now() - now)}ms${
- spanId ? ` (${spanId})` : ''
- }`
- );
+ response$.pipe(concatenateChatCompletionChunks(), lastOperator()).subscribe({
+ error: (error) => {
+ this.dependencies.logger.debug('Error in chat response');
+ this.dependencies.logger.debug(error);
+ },
+ next: (message) => {
+ this.dependencies.logger.debug(`Received message:\n${JSON.stringify(message)}`);
+ },
});
- lastValueFrom(observable)
- .then(
- () => {
+ lastValueFrom(response$)
+ .then(() => {
span?.setOutcome('success');
- },
- () => {
+ })
+ .catch(() => {
span?.setOutcome('failure');
- }
- )
- .finally(() => {
- this.dependencies.logger.debug(
- `Completed response in ${Math.round(performance.now() - now)}ms${
- spanId ? ` (${spanId})` : ''
- }`
- );
-
- span?.end();
- });
+ })
+ .finally(() => {
+ span?.end();
+ });
- return observable;
+ return response$;
+ } catch (error) {
+ span?.setOutcome('failure');
+ span?.end();
+ throw error;
+ }
};
find = async (options?: { query?: string }): Promise<{ conversations: Conversation[] }> => {
@@ -631,13 +623,36 @@ export class ObservabilityAIAssistantClient {
}) => {
const response$ = await this.chat('generate_title', {
messages: [
+ {
+ '@timestamp': new Date().toString(),
+ message: {
+ role: MessageRole.System,
+ content: `You are a helpful assistant for Elastic Observability. Assume the following message is the start of a conversation between you and a user; give this conversation a title based on the content below. DO NOT UNDER ANY CIRCUMSTANCES wrap this title in single or double quotes. This title is shown in a list of conversations to the user, so title it for the user, not for you.`,
+ },
+ },
{
'@timestamp': new Date().toISOString(),
message: {
role: MessageRole.User,
content: messages.slice(1).reduce((acc, curr) => {
return `${acc} ${curr.message.role}: ${curr.message.content}`;
- }, 'You are a helpful assistant for Elastic Observability. Assume the following message is the start of a conversation between you and a user; give this conversation a title based on the content below. DO NOT UNDER ANY CIRCUMSTANCES wrap this title in single or double quotes. This title is shown in a list of conversations to the user, so title it for the user, not for you. Here is the content:'),
+ }, 'Generate a title, using the title_conversation_function, based on the following conversation:\n\n'),
+ },
+ },
+ ],
+ functions: [
+ {
+ name: 'title_conversation',
+ description:
+ 'Use this function to title the conversation. Do not wrap the title in quotes',
+ parameters: {
+ type: 'object',
+ properties: {
+ title: {
+ type: 'string',
+ },
+ },
+ required: ['title'],
},
},
],
@@ -647,7 +662,10 @@ export class ObservabilityAIAssistantClient {
const response = await lastValueFrom(response$.pipe(concatenateChatCompletionChunks()));
- const input = response.message?.content || '';
+ const input =
+ (response.message.function_call.name
+ ? JSON.parse(response.message.function_call.arguments).title
+ : response.message?.content) || '';
// This regular expression captures a string enclosed in single or double quotes.
// It extracts the string content without the quotes.
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.test.ts
new file mode 100644
index 0000000000000..8d1d64721abc4
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.test.ts
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { convertDeserializedXmlWithJsonSchema } from './convert_deserialized_xml_with_json_schema';
+
+describe('deserializeXmlWithJsonSchema', () => {
+ it('deserializes XML into a JSON object according to the JSON schema', () => {
+ expect(
+ convertDeserializedXmlWithJsonSchema(
+ [
+ {
+ foo: ['bar'],
+ },
+ ],
+ {
+ type: 'object',
+ properties: {
+ foo: {
+ type: 'string',
+ },
+ },
+ }
+ )
+ ).toEqual({ foo: 'bar' });
+ });
+
+ it('converts strings to numbers if needed', () => {
+ expect(
+ convertDeserializedXmlWithJsonSchema(
+ [
+ {
+ myNumber: ['0'],
+ },
+ ],
+ {
+ type: 'object',
+ properties: {
+ myNumber: {
+ type: 'number',
+ },
+ },
+ }
+ )
+ ).toEqual({ myNumber: 0 });
+ });
+
+ it('de-dots object paths', () => {
+ expect(
+ convertDeserializedXmlWithJsonSchema(
+ [
+ {
+ 'myObject.foo': ['bar'],
+ },
+ ],
+ {
+ type: 'object',
+ properties: {
+ myObject: {
+ type: 'object',
+ properties: {
+ foo: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ }
+ )
+ ).toEqual({
+ myObject: {
+ foo: 'bar',
+ },
+ });
+ });
+
+ it('casts to an array if needed', () => {
+ expect(
+ convertDeserializedXmlWithJsonSchema(
+ [
+ {
+ myNumber: ['0'],
+ },
+ ],
+ {
+ type: 'object',
+ properties: {
+ myNumber: {
+ type: 'number',
+ },
+ },
+ }
+ )
+ ).toEqual({
+ myNumber: 0,
+ });
+
+ expect(
+ convertDeserializedXmlWithJsonSchema(
+ [
+ {
+ 'labels.myProp': ['myFirstValue, mySecondValue'],
+ },
+ ],
+ {
+ type: 'object',
+ properties: {
+ labels: {
+ type: 'array',
+ items: {
+ type: 'object',
+ properties: {
+ myProp: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ },
+ }
+ )
+ ).toEqual({
+ labels: [{ myProp: 'myFirstValue' }, { myProp: 'mySecondValue' }],
+ });
+ });
+});
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.ts
new file mode 100644
index 0000000000000..a351edb9a33a1
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/convert_deserialized_xml_with_json_schema.ts
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { set } from '@kbn/safer-lodash-set';
+import { unflatten } from 'flat';
+import type { JSONSchema } from 'json-schema-to-ts';
+import { forEach, get, isPlainObject } from 'lodash';
+import { jsonSchemaToFlatParameters } from './json_schema_to_flat_parameters';
+
+// JS to XML is "lossy", e.g. everything becomes an array and a string,
+// so we need a JSON schema to deserialize it
+
+export function convertDeserializedXmlWithJsonSchema(
+ parameterResults: Array>,
+ schema: JSONSchema
+): Record {
+ const parameters = jsonSchemaToFlatParameters(schema);
+
+ const result: Record = Object.fromEntries(
+ parameterResults.flatMap((parameterResult) => {
+ return Object.keys(parameterResult).map((name) => {
+ return [name, parameterResult[name]];
+ });
+ })
+ );
+
+ parameters.forEach((param) => {
+ const key = param.name;
+ let value: any[] = result[key] ?? [];
+ value = param.array
+ ? String(value)
+ .split(',')
+ .map((val) => val.trim())
+ : value;
+
+ switch (param.type) {
+ case 'number':
+ value = value.map((val) => Number(val));
+ break;
+
+ case 'integer':
+ value = value.map((val) => Math.floor(Number(val)));
+ break;
+
+ case 'boolean':
+ value = value.map((val) => String(val).toLowerCase() === 'true' || val === '1');
+ break;
+ }
+
+ result[key] = param.array ? value : value[0];
+ });
+
+ function getArrayPaths(subSchema: JSONSchema, path: string = ''): string[] {
+ if (typeof subSchema === 'boolean') {
+ return [];
+ }
+
+ if (subSchema.type === 'object') {
+ return Object.keys(subSchema.properties!).flatMap((key) => {
+ return getArrayPaths(subSchema.properties![key], path ? path + '.' + key : key);
+ });
+ }
+
+ if (subSchema.type === 'array') {
+ return [path, ...getArrayPaths(subSchema.items as JSONSchema, path)];
+ }
+
+ return [];
+ }
+
+ const arrayPaths = getArrayPaths(schema);
+
+ const unflattened: Record = unflatten(result);
+
+ arrayPaths.forEach((arrayPath) => {
+ const target: any[] = [];
+ function walk(value: any, path: string) {
+ if (Array.isArray(value)) {
+ value.forEach((val, index) => {
+ if (!target[index]) {
+ target[index] = {};
+ }
+ if (path) {
+ set(target[index], path, val);
+ } else {
+ target[index] = val;
+ }
+ });
+ } else if (isPlainObject(value)) {
+ forEach(value, (val, key) => {
+ walk(val, path ? path + '.' + key : key);
+ });
+ }
+ }
+ const val = get(unflattened, arrayPath);
+
+ walk(val, '');
+
+ set(unflattened, arrayPath, target);
+ });
+
+ return unflattened;
+}
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/eventsource_stream_into_observable.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/eventsource_stream_into_observable.ts
new file mode 100644
index 0000000000000..5ff332128f8ac
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/eventsource_stream_into_observable.ts
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createParser } from 'eventsource-parser';
+import { Readable } from 'node:stream';
+import { Observable } from 'rxjs';
+
+// OpenAI sends server-sent events, so we can use a library
+// to deal with parsing, buffering, unicode etc
+
+export function eventsourceStreamIntoObservable(readable: Readable) {
+ return new Observable((subscriber) => {
+ const parser = createParser((event) => {
+ if (event.type === 'event') {
+ subscriber.next(event.data);
+ }
+ });
+
+ async function processStream() {
+ for await (const chunk of readable) {
+ parser.feed(chunk.toString());
+ }
+ }
+
+ processStream().then(
+ () => {
+ subscriber.complete();
+ },
+ (error) => {
+ subscriber.error(error);
+ }
+ );
+ });
+}
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/eventstream_serde_into_observable.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/eventstream_serde_into_observable.ts
new file mode 100644
index 0000000000000..9252ec7588e3e
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/eventstream_serde_into_observable.ts
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EventStreamMarshaller } from '@smithy/eventstream-serde-node';
+import { fromUtf8, toUtf8 } from '@smithy/util-utf8';
+import { identity } from 'lodash';
+import { Observable } from 'rxjs';
+import { Readable } from 'stream';
+import { Message } from '@smithy/types';
+
+interface ModelStreamErrorException {
+ name: 'ModelStreamErrorException';
+ originalStatusCode?: number;
+ originalMessage?: string;
+}
+
+export interface BedrockChunkMember {
+ chunk: Message;
+}
+
+export interface ModelStreamErrorExceptionMember {
+ modelStreamErrorException: ModelStreamErrorException;
+}
+
+export type BedrockStreamMember = BedrockChunkMember | ModelStreamErrorExceptionMember;
+
+// AWS uses SerDe to send over serialized data, so we use their
+// @smithy library to parse the stream data
+
+export function eventstreamSerdeIntoObservable(readable: Readable) {
+ return new Observable((subscriber) => {
+ const marshaller = new EventStreamMarshaller({
+ utf8Encoder: toUtf8,
+ utf8Decoder: fromUtf8,
+ });
+
+ async function processStream() {
+ for await (const chunk of marshaller.deserialize(readable, identity)) {
+ if (chunk) {
+ subscriber.next(chunk as BedrockStreamMember);
+ }
+ }
+ }
+
+ processStream().then(
+ () => {
+ subscriber.complete();
+ },
+ (error) => {
+ subscriber.error(error);
+ }
+ );
+ });
+}
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/flush_buffer.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/flush_buffer.ts
new file mode 100644
index 0000000000000..22723f1e49966
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/flush_buffer.ts
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { repeat } from 'lodash';
+import { identity, Observable, OperatorFunction } from 'rxjs';
+import {
+ BufferFlushEvent,
+ StreamingChatResponseEventType,
+ StreamingChatResponseEventWithoutError,
+} from '../../../common/conversation_complete';
+
+// The Cloud proxy currently buffers 4kb or 8kb of data until flushing.
+// This decreases the responsiveness of the streamed response,
+// so we manually insert some data every 250ms if needed to force it
+// to flush.
+
+export function flushBuffer(
+ isCloud: boolean
+): OperatorFunction {
+ if (!isCloud) {
+ return identity;
+ }
+
+ return (source: Observable) =>
+ new Observable((subscriber) => {
+ const cloudProxyBufferSize = 4096;
+ let currentBufferSize: number = 0;
+
+ const flushBufferIfNeeded = () => {
+ if (currentBufferSize && currentBufferSize <= cloudProxyBufferSize) {
+ subscriber.next({
+ data: repeat('0', cloudProxyBufferSize * 2),
+ type: StreamingChatResponseEventType.BufferFlush,
+ });
+ currentBufferSize = 0;
+ }
+ };
+
+ const intervalId = setInterval(flushBufferIfNeeded, 250);
+
+ source.subscribe({
+ next: (value) => {
+ currentBufferSize =
+ currentBufferSize <= cloudProxyBufferSize
+ ? JSON.stringify(value).length + currentBufferSize
+ : cloudProxyBufferSize;
+ subscriber.next(value);
+ },
+ error: (error) => {
+ clearInterval(intervalId);
+ subscriber.error(error);
+ },
+ complete: () => {
+ clearInterval(intervalId);
+ subscriber.complete();
+ },
+ });
+ });
+}
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.test.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.test.ts
new file mode 100644
index 0000000000000..afcfedf71dc85
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.test.ts
@@ -0,0 +1,208 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { jsonSchemaToFlatParameters } from './json_schema_to_flat_parameters';
+
+describe('jsonSchemaToFlatParameters', () => {
+ it('converts a simple object', () => {
+ expect(
+ jsonSchemaToFlatParameters({
+ type: 'object',
+ properties: {
+ str: {
+ type: 'string',
+ },
+ bool: {
+ type: 'boolean',
+ },
+ },
+ })
+ ).toEqual([
+ {
+ name: 'str',
+ type: 'string',
+ required: false,
+ },
+ {
+ name: 'bool',
+ type: 'boolean',
+ required: false,
+ },
+ ]);
+ });
+
+ it('handles descriptions', () => {
+ expect(
+ jsonSchemaToFlatParameters({
+ type: 'object',
+ properties: {
+ str: {
+ type: 'string',
+ description: 'My string',
+ },
+ },
+ })
+ ).toEqual([
+ {
+ name: 'str',
+ type: 'string',
+ required: false,
+ description: 'My string',
+ },
+ ]);
+ });
+
+ it('handles required properties', () => {
+ expect(
+ jsonSchemaToFlatParameters({
+ type: 'object',
+ properties: {
+ str: {
+ type: 'string',
+ },
+ bool: {
+ type: 'boolean',
+ },
+ },
+ required: ['str'],
+ })
+ ).toEqual([
+ {
+ name: 'str',
+ type: 'string',
+ required: true,
+ },
+ {
+ name: 'bool',
+ type: 'boolean',
+ required: false,
+ },
+ ]);
+ });
+
+ it('handles objects', () => {
+ expect(
+ jsonSchemaToFlatParameters({
+ type: 'object',
+ properties: {
+ nested: {
+ type: 'object',
+ properties: {
+ str: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ required: ['str'],
+ })
+ ).toEqual([
+ {
+ name: 'nested.str',
+ required: false,
+ type: 'string',
+ },
+ ]);
+ });
+
+ it('handles arrays', () => {
+ expect(
+ jsonSchemaToFlatParameters({
+ type: 'object',
+ properties: {
+ arr: {
+ type: 'array',
+ items: {
+ type: 'string',
+ },
+ },
+ },
+ required: ['str'],
+ })
+ ).toEqual([
+ {
+ name: 'arr',
+ required: false,
+ array: true,
+ type: 'string',
+ },
+ ]);
+
+ expect(
+ jsonSchemaToFlatParameters({
+ type: 'object',
+ properties: {
+ arr: {
+ type: 'array',
+ items: {
+ type: 'object',
+ properties: {
+ foo: {
+ type: 'string',
+ },
+ bar: {
+ type: 'object',
+ properties: {
+ baz: {
+ type: 'string',
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ required: ['arr.foo.bar'],
+ })
+ ).toEqual([
+ {
+ name: 'arr.foo',
+ required: false,
+ array: true,
+ type: 'string',
+ },
+ {
+ name: 'arr.bar.baz',
+ required: false,
+ array: true,
+ type: 'string',
+ },
+ ]);
+ });
+
+ it('handles enum and const', () => {
+ expect(
+ jsonSchemaToFlatParameters({
+ type: 'object',
+ properties: {
+ constant: {
+ type: 'string',
+ const: 'foo',
+ },
+ enum: {
+ type: 'number',
+ enum: ['foo', 'bar'],
+ },
+ },
+ required: ['str'],
+ })
+ ).toEqual([
+ {
+ name: 'constant',
+ required: false,
+ type: 'string',
+ constant: 'foo',
+ },
+ {
+ name: 'enum',
+ required: false,
+ type: 'number',
+ enum: ['foo', 'bar'],
+ },
+ ]);
+ });
+});
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.ts
new file mode 100644
index 0000000000000..cd984b0cfd7d0
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/json_schema_to_flat_parameters.ts
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { JSONSchema } from 'json-schema-to-ts';
+import { castArray, isArray } from 'lodash';
+
+interface Parameter {
+ name: string;
+ type: string;
+ description?: string;
+ required?: boolean;
+ enum?: unknown[];
+ constant?: unknown;
+ array?: boolean;
+}
+
+export function jsonSchemaToFlatParameters(
+ schema: JSONSchema,
+ name: string = '',
+ options: { required?: boolean; array?: boolean } = {}
+): Parameter[] {
+ if (typeof schema === 'boolean') {
+ return [];
+ }
+
+ switch (schema.type) {
+ case 'string':
+ case 'number':
+ case 'boolean':
+ case 'integer':
+ case 'null':
+ return [
+ {
+ name,
+ type: schema.type,
+ description: schema.description,
+ array: options.array,
+ required: options.required,
+ constant: schema.const,
+ enum: schema.enum !== undefined ? castArray(schema.enum) : schema.enum,
+ },
+ ];
+
+ case 'array':
+ if (
+ typeof schema.items === 'boolean' ||
+ typeof schema.items === 'undefined' ||
+ isArray(schema.items)
+ ) {
+ return [];
+ }
+ return jsonSchemaToFlatParameters(schema.items as JSONSchema, name, {
+ ...options,
+ array: true,
+ });
+
+ default:
+ case 'object':
+ if (typeof schema.properties === 'undefined') {
+ return [];
+ }
+ return Object.entries(schema.properties).flatMap(([key, subSchema]) => {
+ return jsonSchemaToFlatParameters(subSchema, name ? `${name}.${key}` : key, {
+ ...options,
+ required: schema.required && schema.required.includes(key) ? true : false,
+ });
+ });
+ }
+}
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts
index a1ec52918453f..3ca09acde2b6f 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/observable_into_stream.ts
@@ -8,14 +8,15 @@
import { Observable } from 'rxjs';
import { PassThrough } from 'stream';
import {
+ BufferFlushEvent,
ChatCompletionErrorEvent,
isChatCompletionError,
- StreamingChatResponseEvent,
StreamingChatResponseEventType,
+ StreamingChatResponseEventWithoutError,
} from '../../../common/conversation_complete';
export function observableIntoStream(
- source: Observable>
+ source: Observable
) {
const stream = new PassThrough();
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts
index 764e39fdec152..b2c65c51da9cc 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/stream_into_observable.ts
@@ -5,20 +5,25 @@
* 2.0.
*/
-import { concatMap, filter, from, map, Observable } from 'rxjs';
+import { Observable } from 'rxjs';
import type { Readable } from 'stream';
-export function streamIntoObservable(readable: Readable): Observable {
- let lineBuffer = '';
+export function streamIntoObservable(readable: Readable): Observable {
+ return new Observable((subscriber) => {
+ const decodedStream = readable;
- return from(readable).pipe(
- map((chunk: Buffer) => chunk.toString('utf-8')),
- map((part) => {
- const lines = (lineBuffer + part).split('\n');
- lineBuffer = lines.pop() || ''; // Keep the last incomplete line for the next chunk
- return lines;
- }),
- concatMap((lines) => lines),
- filter((line) => line.trim() !== '')
- );
+ async function processStream() {
+ for await (const chunk of decodedStream) {
+ subscriber.next(chunk);
+ }
+ }
+
+ processStream()
+ .then(() => {
+ subscriber.complete();
+ })
+ .catch((error) => {
+ subscriber.error(error);
+ });
+ });
}
diff --git a/x-pack/plugins/observability_ai_assistant/server/types.ts b/x-pack/plugins/observability_ai_assistant/server/types.ts
index ea2d3ee39e426..21fcc21f39a65 100644
--- a/x-pack/plugins/observability_ai_assistant/server/types.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/types.ts
@@ -23,7 +23,8 @@ import type {
} from '@kbn/data-views-plugin/server';
import type { MlPluginSetup, MlPluginStart } from '@kbn/ml-plugin/server';
import type { LicensingPluginSetup, LicensingPluginStart } from '@kbn/licensing-plugin/server';
-import { ObservabilityAIAssistantService } from './service';
+import type { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server';
+import type { ObservabilityAIAssistantService } from './service';
export interface ObservabilityAIAssistantPluginSetup {
/**
@@ -47,6 +48,7 @@ export interface ObservabilityAIAssistantPluginSetupDependencies {
dataViews: DataViewsServerPluginSetup;
ml: MlPluginSetup;
licensing: LicensingPluginSetup;
+ cloud?: CloudSetup;
}
export interface ObservabilityAIAssistantPluginStartDependencies {
actions: ActionsPluginStart;
@@ -56,4 +58,5 @@ export interface ObservabilityAIAssistantPluginStartDependencies {
dataViews: DataViewsServerPluginStart;
ml: MlPluginStart;
licensing: LicensingPluginStart;
+ cloud?: CloudStart;
}
diff --git a/x-pack/plugins/observability_ai_assistant/tsconfig.json b/x-pack/plugins/observability_ai_assistant/tsconfig.json
index f5a29c470fe7a..13af731fd49db 100644
--- a/x-pack/plugins/observability_ai_assistant/tsconfig.json
+++ b/x-pack/plugins/observability_ai_assistant/tsconfig.json
@@ -61,6 +61,8 @@
"@kbn/apm-synthtrace-client",
"@kbn/apm-synthtrace",
"@kbn/code-editor",
+ "@kbn/safer-lodash-set",
+ "@kbn/cloud-plugin",
"@kbn/ui-actions-plugin",
"@kbn/expressions-plugin",
"@kbn/visualization-utils",
diff --git a/x-pack/plugins/stack_connectors/common/bedrock/constants.ts b/x-pack/plugins/stack_connectors/common/bedrock/constants.ts
index 242447d505218..ea3fb7af72fa9 100644
--- a/x-pack/plugins/stack_connectors/common/bedrock/constants.ts
+++ b/x-pack/plugins/stack_connectors/common/bedrock/constants.ts
@@ -23,6 +23,6 @@ export enum SUB_ACTION {
}
export const DEFAULT_TOKEN_LIMIT = 8191;
-export const DEFAULT_BEDROCK_MODEL = 'anthropic.claude-v2';
+export const DEFAULT_BEDROCK_MODEL = 'anthropic.claude-v2:1';
export const DEFAULT_BEDROCK_URL = `https://bedrock-runtime.us-east-1.amazonaws.com` as const;
diff --git a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts
index 057780a803560..c26ce8c1e88c3 100644
--- a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts
+++ b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts
@@ -32,6 +32,8 @@ export const InvokeAIActionParamsSchema = schema.object({
})
),
model: schema.maybe(schema.string()),
+ temperature: schema.maybe(schema.number()),
+ stopSequences: schema.maybe(schema.arrayOf(schema.string())),
});
export const InvokeAIActionResponseSchema = schema.object({
diff --git a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx
index 7678f52321dd3..0ccd8c1d08023 100644
--- a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx
+++ b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/params.tsx
@@ -102,7 +102,7 @@ const BedrockParamsFields: React.FunctionComponent
{
editSubActionParams({ model: ev.target.value });
diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts
index 3cd2ad2061ffd..3b1cb3bc96ec8 100644
--- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts
+++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts
@@ -73,7 +73,7 @@ describe('BedrockConnector', () => {
'Content-Type': 'application/json',
},
host: 'bedrock-runtime.us-east-1.amazonaws.com',
- path: '/model/anthropic.claude-v2/invoke',
+ path: '/model/anthropic.claude-v2:1/invoke',
service: 'bedrock',
},
{ accessKeyId: '123', secretAccessKey: 'secret' }
@@ -137,7 +137,7 @@ describe('BedrockConnector', () => {
'x-amzn-bedrock-accept': '*/*',
},
host: 'bedrock-runtime.us-east-1.amazonaws.com',
- path: '/model/anthropic.claude-v2/invoke-with-response-stream',
+ path: '/model/anthropic.claude-v2:1/invoke-with-response-stream',
service: 'bedrock',
},
{ accessKeyId: '123', secretAccessKey: 'secret' }
@@ -165,14 +165,53 @@ describe('BedrockConnector', () => {
it('formats messages from user, assistant, and system', async () => {
await connector.invokeStream({
messages: [
+ {
+ role: 'system',
+ content: 'Be a good chatbot',
+ },
{
role: 'user',
content: 'Hello world',
},
+ {
+ role: 'assistant',
+ content: 'Hi, I am a good chatbot',
+ },
+ {
+ role: 'user',
+ content: 'What is 2+2?',
+ },
+ ],
+ });
+ expect(mockRequest).toHaveBeenCalledWith({
+ signed: true,
+ responseType: 'stream',
+ url: `${DEFAULT_BEDROCK_URL}/model/${DEFAULT_BEDROCK_MODEL}/invoke-with-response-stream`,
+ method: 'post',
+ responseSchema: StreamingResponseSchema,
+ data: JSON.stringify({
+ prompt:
+ 'Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:',
+ max_tokens_to_sample: DEFAULT_TOKEN_LIMIT,
+ temperature: 0.5,
+ stop_sequences: ['\n\nHuman:'],
+ }),
+ });
+ });
+
+ it('formats the system message as a user message for claude<2.1', async () => {
+ const modelOverride = 'anthropic.claude-v2';
+
+ await connector.invokeStream({
+ messages: [
{
role: 'system',
content: 'Be a good chatbot',
},
+ {
+ role: 'user',
+ content: 'Hello world',
+ },
{
role: 'assistant',
content: 'Hi, I am a good chatbot',
@@ -182,16 +221,17 @@ describe('BedrockConnector', () => {
content: 'What is 2+2?',
},
],
+ model: modelOverride,
});
expect(mockRequest).toHaveBeenCalledWith({
signed: true,
responseType: 'stream',
- url: `${DEFAULT_BEDROCK_URL}/model/${DEFAULT_BEDROCK_MODEL}/invoke-with-response-stream`,
+ url: `${DEFAULT_BEDROCK_URL}/model/${modelOverride}/invoke-with-response-stream`,
method: 'post',
responseSchema: StreamingResponseSchema,
data: JSON.stringify({
prompt:
- '\n\nHuman:Hello world\n\nHuman:Be a good chatbot\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:',
+ '\n\nHuman:Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:',
max_tokens_to_sample: DEFAULT_TOKEN_LIMIT,
temperature: 0.5,
stop_sequences: ['\n\nHuman:'],
@@ -244,14 +284,14 @@ describe('BedrockConnector', () => {
it('formats messages from user, assistant, and system', async () => {
const response = await connector.invokeAI({
messages: [
- {
- role: 'user',
- content: 'Hello world',
- },
{
role: 'system',
content: 'Be a good chatbot',
},
+ {
+ role: 'user',
+ content: 'Hello world',
+ },
{
role: 'assistant',
content: 'Hi, I am a good chatbot',
@@ -271,7 +311,7 @@ describe('BedrockConnector', () => {
responseSchema: RunActionResponseSchema,
data: JSON.stringify({
prompt:
- '\n\nHuman:Hello world\n\nHuman:Be a good chatbot\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:',
+ 'Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:',
max_tokens_to_sample: DEFAULT_TOKEN_LIMIT,
temperature: 0.5,
stop_sequences: ['\n\nHuman:'],
diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts
index f70a592509776..3fdbaae1d702a 100644
--- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts
+++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts
@@ -26,7 +26,11 @@ import type {
InvokeAIActionResponse,
StreamActionParams,
} from '../../../common/bedrock/types';
-import { SUB_ACTION, DEFAULT_TOKEN_LIMIT } from '../../../common/bedrock/constants';
+import {
+ SUB_ACTION,
+ DEFAULT_TOKEN_LIMIT,
+ DEFAULT_BEDROCK_MODEL,
+} from '../../../common/bedrock/constants';
import {
DashboardActionParams,
DashboardActionResponse,
@@ -233,9 +237,14 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B
* @param messages An array of messages to be sent to the API
* @param model Optional model to be used for the API request. If not provided, the default model from the connector will be used.
*/
- public async invokeStream({ messages, model }: InvokeAIActionParams): Promise {
+ public async invokeStream({
+ messages,
+ model,
+ stopSequences,
+ temperature,
+ }: InvokeAIActionParams): Promise {
const res = (await this.streamApi({
- body: JSON.stringify(formatBedrockBody({ messages })),
+ body: JSON.stringify(formatBedrockBody({ messages, model, stopSequences, temperature })),
model,
})) as unknown as IncomingMessage;
return res;
@@ -250,20 +259,43 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B
messages,
model,
}: InvokeAIActionParams): Promise {
- const res = await this.runApi({ body: JSON.stringify(formatBedrockBody({ messages })), model });
+ const res = await this.runApi({
+ body: JSON.stringify(formatBedrockBody({ messages, model })),
+ model,
+ });
return { message: res.completion.trim() };
}
}
const formatBedrockBody = ({
+ model = DEFAULT_BEDROCK_MODEL,
messages,
+ stopSequences = ['\n\nHuman:'],
+ temperature = 0.5,
}: {
+ model?: string;
messages: Array<{ role: string; content: string }>;
+ stopSequences?: string[];
+ temperature?: number;
}) => {
const combinedMessages = messages.reduce((acc: string, message) => {
const { role, content } = message;
- // Bedrock only has Assistant and Human, so 'system' and 'user' will be converted to Human
- const bedrockRole = role === 'assistant' ? '\n\nAssistant:' : '\n\nHuman:';
+ const [, , modelName, majorVersion, minorVersion] =
+ (model || '').match(/(\w+)\.(.*)-v(\d+)(?::(\d+))?/) || [];
+ // Claude only has Assistant and Human, so 'user' will be converted to Human
+ let bedrockRole: string;
+
+ if (
+ role === 'system' &&
+ modelName === 'claude' &&
+ Number(majorVersion) >= 2 &&
+ Number(minorVersion) >= 1
+ ) {
+ bedrockRole = '';
+ } else {
+ bedrockRole = role === 'assistant' ? '\n\nAssistant:' : '\n\nHuman:';
+ }
+
return `${acc}${bedrockRole}${content}`;
}, '');
@@ -271,8 +303,8 @@ const formatBedrockBody = ({
// end prompt in "Assistant:" to avoid the model starting its message with "Assistant:"
prompt: `${combinedMessages} \n\nAssistant:`,
max_tokens_to_sample: DEFAULT_TOKEN_LIMIT,
- temperature: 0.5,
+ temperature,
// prevent model from talking to itself
- stop_sequences: ['\n\nHuman:'],
+ stop_sequences: stopSequences,
};
};
diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts
index 5f295b8c39367..688148d51ed63 100644
--- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts
+++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts
@@ -10,7 +10,10 @@ import {
SubActionConnectorType,
ValidatorType,
} from '@kbn/actions-plugin/server/sub_action_framework/types';
-import { GenerativeAIForSecurityConnectorFeatureId } from '@kbn/actions-plugin/common';
+import {
+ GenerativeAIForObservabilityConnectorFeatureId,
+ GenerativeAIForSecurityConnectorFeatureId,
+} from '@kbn/actions-plugin/common';
import { urlAllowListValidator } from '@kbn/actions-plugin/server';
import { ValidatorServices } from '@kbn/actions-plugin/server/types';
import { assertURL } from '@kbn/actions-plugin/server/sub_action_framework/helpers/validators';
@@ -29,7 +32,10 @@ export const getConnectorType = (): SubActionConnectorType => (
secrets: SecretsSchema,
},
validators: [{ type: ValidatorType.CONFIG, validator: configValidator }],
- supportedFeatureIds: [GenerativeAIForSecurityConnectorFeatureId],
+ supportedFeatureIds: [
+ GenerativeAIForSecurityConnectorFeatureId,
+ GenerativeAIForObservabilityConnectorFeatureId,
+ ],
minimumLicenseRequired: 'enterprise' as const,
renderParameterTemplates,
});
diff --git a/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts b/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts
index 29e77feb5edaf..18051754cc77a 100644
--- a/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts
+++ b/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/bedrock_simulation.ts
@@ -29,7 +29,7 @@ export class BedrockSimulator extends Simulator {
return BedrockSimulator.sendErrorResponse(response);
}
- if (request.url === '/model/anthropic.claude-v2/invoke-with-response-stream') {
+ if (request.url === '/model/anthropic.claude-v2:1/invoke-with-response-stream') {
return BedrockSimulator.sendStreamResponse(response);
}
diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts
index 14ec27598a60f..fc0ca3378d8c0 100644
--- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts
+++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts
@@ -27,7 +27,7 @@ const secrets = {
};
const defaultConfig = {
- defaultModel: 'anthropic.claude-v2',
+ defaultModel: 'anthropic.claude-v2:1',
};
// eslint-disable-next-line import/no-default-export
@@ -380,14 +380,14 @@ export default function bedrockTest({ getService }: FtrProviderContext) {
subAction: 'invokeAI',
subActionParams: {
messages: [
- {
- role: 'user',
- content: 'Hello world',
- },
{
role: 'system',
content: 'Be a good chatbot',
},
+ {
+ role: 'user',
+ content: 'Hello world',
+ },
{
role: 'assistant',
content: 'Hi, I am a good chatbot',
@@ -404,7 +404,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) {
expect(simulator.requestData).to.eql({
prompt:
- '\n\nHuman:Hello world\n\nHuman:Be a good chatbot\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:',
+ 'Be a good chatbot\n\nHuman:Hello world\n\nAssistant:Hi, I am a good chatbot\n\nHuman:What is 2+2? \n\nAssistant:',
max_tokens_to_sample: DEFAULT_TOKEN_LIMIT,
temperature: 0.5,
stop_sequences: ['\n\nHuman:'],
diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts b/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts
index 3aaaf982c3597..aead4e6276c56 100644
--- a/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts
+++ b/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts
@@ -65,7 +65,8 @@ export class LlmProxy {
}
}
- throw new Error('No interceptors found to handle request');
+ response.writeHead(500, 'No interceptors found to handle request: ' + request.url);
+ response.end();
})
.listen(port);
}
@@ -111,7 +112,7 @@ export class LlmProxy {
}),
next: (msg) => {
const chunk = createOpenAiChunk(msg);
- return write(`data: ${JSON.stringify(chunk)}\n`);
+ return write(`data: ${JSON.stringify(chunk)}\n\n`);
},
rawWrite: (chunk: string) => {
return write(chunk);
@@ -120,11 +121,11 @@ export class LlmProxy {
await end();
},
complete: async () => {
- await write('data: [DONE]');
+ await write('data: [DONE]\n\n');
await end();
},
error: async (error) => {
- await write(`data: ${JSON.stringify({ error })}`);
+ await write(`data: ${JSON.stringify({ error })}\n\n`);
await end();
},
};
diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts
index c56af40a6ab29..82ad5b6dd1224 100644
--- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts
+++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts
@@ -104,7 +104,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const chunk = JSON.stringify(createOpenAiChunk('Hello'));
await simulator.rawWrite(`data: ${chunk.substring(0, 10)}`);
- await simulator.rawWrite(`${chunk.substring(10)}\n`);
+ await simulator.rawWrite(`${chunk.substring(10)}\n\n`);
await simulator.complete();
await new Promise((resolve) => passThrough.on('end', () => resolve()));
@@ -146,15 +146,17 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const titleInterceptor = proxy.intercept(
'title',
(body) =>
- (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming).messages
- .length === 1
+ (
+ JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming
+ ).functions?.find((fn) => fn.name === 'title_conversation') !== undefined
);
const conversationInterceptor = proxy.intercept(
'conversation',
(body) =>
- (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming).messages
- .length !== 1
+ (
+ JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming
+ ).functions?.find((fn) => fn.name === 'title_conversation') === undefined
);
const responsePromise = new Promise((resolve, reject) => {
diff --git a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts
index ce9fc050b5e09..c0b2b36dfc029 100644
--- a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts
+++ b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts
@@ -148,15 +148,17 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
const titleInterceptor = proxy.intercept(
'title',
(body) =>
- (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming)
- .messages.length === 1
+ (
+ JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming
+ ).functions?.find((fn) => fn.name === 'title_conversation') !== undefined
);
const conversationInterceptor = proxy.intercept(
'conversation',
(body) =>
- (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming)
- .messages.length !== 1
+ (
+ JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming
+ ).functions?.find((fn) => fn.name === 'title_conversation') === undefined
);
await testSubjects.setValue(ui.pages.conversations.chatInput, 'hello');
diff --git a/yarn.lock b/yarn.lock
index 62b6925bfe980..42e12233631a3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7750,14 +7750,32 @@
"@types/node" ">=18.0.0"
axios "^1.6.0"
-"@smithy/eventstream-codec@^2.0.12":
- version "2.0.12"
- resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-2.0.12.tgz#99fab750d0ac3941f341d912d3c3a1ab985e1a7a"
- integrity sha512-ZZQLzHBJkbiAAdj2C5K+lBlYp/XJ+eH2uy+jgJgYIFW/o5AM59Hlj7zyI44/ZTDIQWmBxb3EFv/c5t44V8/g8A==
+"@smithy/eventstream-codec@^2.0.12", "@smithy/eventstream-codec@^2.1.1":
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-2.1.1.tgz#4405ab0f9c77d439c575560c4886e59ee17d6d38"
+ integrity sha512-E8KYBxBIuU4c+zrpR22VsVrOPoEDzk35bQR3E+xm4k6Pa6JqzkDOdMyf9Atac5GPNKHJBdVaQ4JtjdWX2rl/nw==
dependencies:
"@aws-crypto/crc32" "3.0.0"
- "@smithy/types" "^2.4.0"
- "@smithy/util-hex-encoding" "^2.0.0"
+ "@smithy/types" "^2.9.1"
+ "@smithy/util-hex-encoding" "^2.1.1"
+ tslib "^2.5.0"
+
+"@smithy/eventstream-serde-node@^2.1.1":
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-node/-/eventstream-serde-node-2.1.1.tgz#2e1afa27f9c7eb524c1c53621049c5e4e3cea6a5"
+ integrity sha512-LF882q/aFidFNDX7uROAGxq3H0B7rjyPkV6QDn6/KDQ+CG7AFkRccjxRf1xqajq/Pe4bMGGr+VKAaoF6lELIQw==
+ dependencies:
+ "@smithy/eventstream-serde-universal" "^2.1.1"
+ "@smithy/types" "^2.9.1"
+ tslib "^2.5.0"
+
+"@smithy/eventstream-serde-universal@^2.1.1":
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-2.1.1.tgz#0f5eec9ad033017973a67bafb5549782499488d2"
+ integrity sha512-LR0mMT+XIYTxk4k2fIxEA1BPtW3685QlqufUEUAX1AJcfFfxNDKEvuCRZbO8ntJb10DrIFVJR9vb0MhDCi0sAQ==
+ dependencies:
+ "@smithy/eventstream-codec" "^2.1.1"
+ "@smithy/types" "^2.9.1"
tslib "^2.5.0"
"@smithy/is-array-buffer@^2.0.0":
@@ -7767,10 +7785,10 @@
dependencies:
tslib "^2.5.0"
-"@smithy/types@^2.4.0":
- version "2.4.0"
- resolved "https://registry.yarnpkg.com/@smithy/types/-/types-2.4.0.tgz#ed35e429e3ea3d089c68ed1bf951d0ccbdf2692e"
- integrity sha512-iH1Xz68FWlmBJ9vvYeHifVMWJf82ONx+OybPW8ZGf5wnEv2S0UXcU4zwlwJkRXuLKpcSLHrraHbn2ucdVXLb4g==
+"@smithy/types@^2.4.0", "@smithy/types@^2.9.1":
+ version "2.9.1"
+ resolved "https://registry.yarnpkg.com/@smithy/types/-/types-2.9.1.tgz#ed04d4144eed3b8bd26d20fc85aae8d6e357ebb9"
+ integrity sha512-vjXlKNXyprDYDuJ7UW5iobdmyDm6g8dDG+BFUncAg/3XJaN45Gy5RWWWUVgrzIK7S4R1KWgIX5LeJcfvSI24bw==
dependencies:
tslib "^2.5.0"
@@ -7782,10 +7800,10 @@
"@smithy/is-array-buffer" "^2.0.0"
tslib "^2.5.0"
-"@smithy/util-hex-encoding@^2.0.0":
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/@smithy/util-hex-encoding/-/util-hex-encoding-2.0.0.tgz#0aa3515acd2b005c6d55675e377080a7c513b59e"
- integrity sha512-c5xY+NUnFqG6d7HFh1IFfrm3mGl29lC+vF+geHv4ToiuJCBmIfzx6IeHLg+OgRdPFKDXIw6pvi+p3CsscaMcMA==
+"@smithy/util-hex-encoding@^2.1.1":
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/@smithy/util-hex-encoding/-/util-hex-encoding-2.1.1.tgz#978252b9fb242e0a59bae4ead491210688e0d15f"
+ integrity sha512-3UNdP2pkYUUBGEXzQI9ODTDK+Tcu1BlCyDBaRHwyxhA+8xLP8agEKQq4MGmpjqb4VQAjq9TwlCQX0kP6XDKYLg==
dependencies:
tslib "^2.5.0"
@@ -9346,6 +9364,13 @@
resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.0.tgz#5fb2e536c1ae9bf35366eed879e827fa59ca41c2"
integrity sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ==
+"@types/event-stream@^4.0.5":
+ version "4.0.5"
+ resolved "https://registry.yarnpkg.com/@types/event-stream/-/event-stream-4.0.5.tgz#29f1be5f4c0de2e0312cf3b5f7146c975c08d918"
+ integrity sha512-pQ/RR/iuBW8K8WmwYaaC1nkZH0cHonNAIw6ktG8BCNrNuqNeERfBzNIAOq6Z7tvLzpjcMV02SZ5pxAekAYQpWA==
+ dependencies:
+ "@types/node" "*"
+
"@types/expect@^1.20.4":
version "1.20.4"
resolved "https://registry.yarnpkg.com/@types/expect/-/expect-1.20.4.tgz#8288e51737bf7e3ab5d7c77bfa695883745264e5"
@@ -9390,6 +9415,11 @@
resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.0.tgz#cbb49815a5e1129d5f23836a98d65d93822409af"
integrity sha512-dxdRrUov2HVTbSRFX+7xwUPlbGYVEZK6PrSqClg2QPos3PNe0bCajkDDkDeeC1znjSH03KOEqVbXpnJuWa2wgQ==
+"@types/flat@^5.0.5":
+ version "5.0.5"
+ resolved "https://registry.yarnpkg.com/@types/flat/-/flat-5.0.5.tgz#2304df0b2b1e6dde50d81f029593e0a1bc2474d3"
+ integrity sha512-nPLljZQKSnac53KDUDzuzdRfGI0TDb5qPrb+SrQyN3MtdQrOnGsKniHN1iYZsJEBIVQve94Y6gNz22sgISZq+Q==
+
"@types/flot@^0.0.31":
version "0.0.31"
resolved "https://registry.yarnpkg.com/@types/flot/-/flot-0.0.31.tgz#0daca37c6c855b69a0a7e2e37dd0f84b3db8c8c1"
@@ -16620,6 +16650,11 @@ events@^3.0.0, events@^3.2.0, events@^3.3.0:
resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==
+eventsource-parser@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/eventsource-parser/-/eventsource-parser-1.1.1.tgz#576f8bcf391c5e5ccdea817abd9ead36d1754247"
+ integrity sha512-3Ej2iLj6ZnX+5CMxqyUb8syl9yVZwcwm8IIMrOJlF7I51zxOOrRlU3zxSb/6hFbl03ts1ZxHAGJdWLZOLyKG7w==
+
evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02"
@@ -17272,7 +17307,7 @@ flat-cache@^3.0.4:
flatted "^3.1.0"
rimraf "^3.0.2"
-flat@^5.0.2:
+flat@5, flat@^5.0.2:
version "5.0.2"
resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241"
integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==
From e20e6598767bb5e9f80b4629b5785b5dc63fc904 Mon Sep 17 00:00:00 2001
From: Ersin Erdal <92688503+ersin-erdal@users.noreply.github.com>
Date: Fri, 9 Feb 2024 09:18:44 +0100
Subject: [PATCH 05/13] Refactor TM to update the tasks that has state
validation error (#176415)
Resolves: #172605
This PR makes TM TaskRunner to handle state validation errors gracefully
to allow it update the task state.
## To verify:
1 - Create a rule with some actions.
2- Throw an error in [state validation
function](https://github.com/elastic/kibana/pull/176415/files#diff-ae4166cd6b3509473867eaed0e7b974a15b9c0268225131aef1b00d61e800e89R428)
to force it to return an error.
3- Expect the rule tasks to run and update the task state successfully
rather than throwing an error and preventing task update.
---
.../task_state_validation.test.ts | 6 ++--
x-pack/plugins/task_manager/server/task.ts | 1 +
.../server/task_running/task_runner.test.ts | 20 +++++++++---
.../server/task_running/task_runner.ts | 32 ++++++++++++++-----
4 files changed, 45 insertions(+), 14 deletions(-)
diff --git a/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts b/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts
index 716e1f8dcb83f..c7ee109d17b11 100644
--- a/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts
+++ b/x-pack/plugins/task_manager/server/integration_tests/task_state_validation.test.ts
@@ -308,6 +308,7 @@ describe('task state validation', () => {
it('should fail the task run when setting allow_reading_invalid_state:false and reading an invalid state', async () => {
const logSpy = jest.spyOn(pollingLifecycleOpts.logger, 'warn');
+ const updateSpy = jest.spyOn(pollingLifecycleOpts.taskStore, 'bulkUpdate');
const id = uuidV4();
await injectTask(kibanaServer.coreStart.elasticsearch.client.asInternalUser, {
@@ -331,8 +332,9 @@ describe('task state validation', () => {
expect(logSpy.mock.calls[0][0]).toBe(
`Task (fooType/${id}) has a validation error: [foo]: expected value of type [string] but got [boolean]`
);
- expect(logSpy.mock.calls[1][0]).toBe(
- `Task fooType \"${id}\" failed in attempt to run: [foo]: expected value of type [string] but got [boolean]`
+ expect(updateSpy).toHaveBeenCalledWith(
+ expect.arrayContaining([expect.objectContaining({ id, taskType: 'fooType' })]),
+ { validate: false }
);
});
});
diff --git a/x-pack/plugins/task_manager/server/task.ts b/x-pack/plugins/task_manager/server/task.ts
index c71f8b42185ca..0d064153859a5 100644
--- a/x-pack/plugins/task_manager/server/task.ts
+++ b/x-pack/plugins/task_manager/server/task.ts
@@ -50,6 +50,7 @@ export type SuccessfulRunResult = {
state: Record;
taskRunError?: DecoratedError;
skipAttempts?: number;
+ shouldValidate?: boolean;
} & (
| // ensure a SuccessfulRunResult can either specify a new `runAt` or a new `schedule`, but not both
{
diff --git a/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts b/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts
index 8a96405abfed6..6735b3c0602b8 100644
--- a/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts
+++ b/x-pack/plugins/task_manager/server/task_running/task_runner.test.ts
@@ -1082,6 +1082,7 @@ describe('TaskManagerRunner', () => {
await runner.run();
expect(store.update).toHaveBeenCalledTimes(1);
+ expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true });
const instance = store.update.mock.calls[0][0];
expect(instance.runAt.getTime()).toEqual(nextRetry.getTime());
@@ -1113,6 +1114,8 @@ describe('TaskManagerRunner', () => {
await runner.run();
expect(store.update).toHaveBeenCalledTimes(1);
+ expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true });
+
const instance = store.update.mock.calls[0][0];
const minRunAt = Date.now();
@@ -1179,6 +1182,8 @@ describe('TaskManagerRunner', () => {
await runner.run();
expect(store.update).toHaveBeenCalledTimes(1);
+ expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true });
+
sinon.assert.notCalled(getRetryStub);
const instance = store.update.mock.calls[0][0];
@@ -1252,6 +1257,7 @@ describe('TaskManagerRunner', () => {
new Date(Date.now() + intervalSeconds * 1000).getTime()
);
expect(instance.enabled).not.toBeDefined();
+ expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: true });
});
test('throws error when the task has invalid state', async () => {
@@ -1266,7 +1272,7 @@ describe('TaskManagerRunner', () => {
stateVersion: 4,
};
- const { runner, logger } = await readyToRunStageSetup({
+ const { runner, logger, store } = await readyToRunStageSetup({
instance: mockTaskInstance,
definitions: {
bar: {
@@ -1308,13 +1314,19 @@ describe('TaskManagerRunner', () => {
},
});
- expect(() => runner.run()).rejects.toMatchInlineSnapshot(
- `[Error: [foo]: expected value of type [string] but got [boolean]]`
- );
+ expect(await runner.run()).toEqual({
+ error: {
+ error: new Error('[foo]: expected value of type [string] but got [boolean]'),
+ shouldValidate: false,
+ state: { bar: 'test', baz: 'test', foo: true },
+ },
+ tag: 'err',
+ });
expect(logger.warn).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledWith(
'Task (bar/foo) has a validation error: [foo]: expected value of type [string] but got [boolean]'
);
+ expect(store.update).toHaveBeenCalledWith(expect.any(Object), { validate: false });
});
test('does not throw error and runs when the task has invalid state and allowReadingInvalidState = true', async () => {
diff --git a/x-pack/plugins/task_manager/server/task_running/task_runner.ts b/x-pack/plugins/task_manager/server/task_running/task_runner.ts
index ab86d83e99310..faea2bfb7e446 100644
--- a/x-pack/plugins/task_manager/server/task_running/task_runner.ts
+++ b/x-pack/plugins/task_manager/server/task_running/task_runner.ts
@@ -314,16 +314,30 @@ export class TaskManagerRunner implements TaskRunner {
const apmTrans = apm.startTransaction(this.taskType, TASK_MANAGER_RUN_TRANSACTION_TYPE, {
childOf: this.instance.task.traceparent,
});
+ const stopTaskTimer = startTaskTimerWithEventLoopMonitoring(this.eventLoopDelayConfig);
// Validate state
- const validatedTaskInstance = this.validateTaskState(this.instance.task);
+ const stateValidationResult = this.validateTaskState(this.instance.task);
+
+ if (stateValidationResult.error) {
+ const processedResult = await withSpan({ name: 'process result', type: 'task manager' }, () =>
+ this.processResult(
+ asErr({
+ error: stateValidationResult.error,
+ state: stateValidationResult.taskInstance.state,
+ shouldValidate: false,
+ }),
+ stopTaskTimer()
+ )
+ );
+ if (apmTrans) apmTrans.end('failure');
+ return processedResult;
+ }
const modifiedContext = await this.beforeRun({
- taskInstance: validatedTaskInstance,
+ taskInstance: stateValidationResult.taskInstance,
});
- const stopTaskTimer = startTaskTimerWithEventLoopMonitoring(this.eventLoopDelayConfig);
-
this.onTaskEvent(
asTaskManagerStatEvent(
'runDelay',
@@ -411,11 +425,12 @@ export class TaskManagerRunner implements TaskRunner {
private validateTaskState(taskInstance: ConcreteTaskInstance) {
const { taskType, id } = taskInstance;
try {
- const validatedTask = this.taskValidator.getValidatedTaskInstanceFromReading(taskInstance);
- return validatedTask;
+ const validatedTaskInstance =
+ this.taskValidator.getValidatedTaskInstanceFromReading(taskInstance);
+ return { taskInstance: validatedTaskInstance, error: null };
} catch (error) {
this.logger.warn(`Task (${taskType}/${id}) has a validation error: ${error.message}`);
- throw error;
+ return { taskInstance, error };
}
}
@@ -723,6 +738,7 @@ export class TaskManagerRunner implements TaskRunner {
this.instance = asRan(this.instance.task);
await this.removeTask();
} else {
+ const { shouldValidate = true } = unwrap(result);
this.instance = asRan(
await this.bufferedTaskStore.update(
defaults(
@@ -735,7 +751,7 @@ export class TaskManagerRunner implements TaskRunner {
},
taskWithoutEnabled(this.instance.task)
),
- { validate: true }
+ { validate: shouldValidate }
)
);
}
From dfa053b5a13839cca770460c54c4b47afd76dc94 Mon Sep 17 00:00:00 2001
From: Robert Oskamp
Date: Fri, 9 Feb 2024 09:50:20 +0100
Subject: [PATCH 06/13] Fix serverless test user for MKI runs (#176430)
## Summary
This PR fixes an issue when running FTR tests against MKI with the new
internal test user.
### Details
- The hard-coded `elastic` in the expected username of the
authentication test has been replaced by whatever username is
configured, making it work for local and MKI runs
- During debugging, I've noticed an incomplete cleanup after the index
template tests (running this suite twice on the same project failed due
to an already existing resource). Added a proper cleanup.
---
.../index_management/lib/templates.api.ts | 4 ++--
.../common/index_management/index_templates.ts | 13 ++++++++++++-
.../common/platform_security/authentication.ts | 4 ++--
3 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts
index 6e8fbffbe0416..e929cbff2f188 100644
--- a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts
+++ b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts
@@ -36,9 +36,9 @@ export function templatesApi(getService: FtrProviderContext['getService']) {
.send(payload);
// Delete all templates created during tests
- const cleanUpTemplates = async () => {
+ const cleanUpTemplates = async (additionalRequestHeaders: object = {}) => {
try {
- await deleteTemplates(templatesCreated);
+ await deleteTemplates(templatesCreated).set(additionalRequestHeaders);
templatesCreated = [];
} catch (e) {
// Silently swallow errors
diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts
index 0b94803bcd765..6546d94afe391 100644
--- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts
+++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts
@@ -23,6 +23,7 @@ export default function ({ getService }: FtrProviderContext) {
let updateTemplate: typeof indexManagementService['templates']['api']['updateTemplate'];
let deleteTemplates: typeof indexManagementService['templates']['api']['deleteTemplates'];
let simulateTemplate: typeof indexManagementService['templates']['api']['simulateTemplate'];
+ let cleanUpTemplates: typeof indexManagementService['templates']['api']['cleanUpTemplates'];
let getRandomString: () => string;
describe('Index templates', function () {
@@ -30,12 +31,22 @@ export default function ({ getService }: FtrProviderContext) {
({
templates: {
helpers: { getTemplatePayload, catTemplate, getSerializedTemplate },
- api: { createTemplate, updateTemplate, deleteTemplates, simulateTemplate },
+ api: {
+ createTemplate,
+ updateTemplate,
+ deleteTemplates,
+ simulateTemplate,
+ cleanUpTemplates,
+ },
},
} = indexManagementService);
getRandomString = () => randomness.string({ casing: 'lower', alpha: true });
});
+ after(async () => {
+ await cleanUpTemplates({ 'x-elastic-internal-origin': 'xxx' });
+ });
+
describe('get', () => {
let templateName: string;
diff --git a/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts b/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts
index 4c8353487adce..da71d2ad4858a 100644
--- a/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts
+++ b/x-pack/test_serverless/api_integration/test_suites/common/platform_security/authentication.ts
@@ -11,6 +11,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService }: FtrProviderContext) {
const svlCommonApi = getService('svlCommonApi');
const supertest = getService('supertest');
+ const config = getService('config');
describe('security/authentication', function () {
describe('route access', () => {
@@ -144,8 +145,7 @@ export default function ({ getService }: FtrProviderContext) {
metadata: {},
operator: true,
roles: ['superuser'],
- // We use `elastic` for MKI, and `elastic_serverless` for any other testing environment.
- username: expect.stringContaining('elastic'),
+ username: config.get('servers.kibana.username'),
});
expect(status).toBe(200);
});
From d2f2d43b7f691d529811e46afa2cd9791c9fc0c5 Mon Sep 17 00:00:00 2001
From: Pablo Machado
Date: Fri, 9 Feb 2024 09:51:15 +0100
Subject: [PATCH 07/13] [Security Solution] Improve alert UX for criticality
enrichments (#176056)
## Summary
### Assign user-friendly column labels to enrichment fields for improved
readability.
* Add a `displayAsText` prop for Entity Analytics fields on the Alerts
table.
* host.asset.criticality -> Host Criticality
* user.asset.criticality -> User Criticality
* host.risk.calculated.level -> Host Risk Level
* user.risk.calculated.level -> User Risk Level
* Add migration that renames `host.risk.calculated._evel` and
`user.risk.calculated_level` on local storage
### Render a custom component for the Host and User criticality inside
the Alerts table cell.
### How to test it?
* Clone and install
https://github.com/elastic/security-documents-generator
* Execute `npm start entity-store` on an empty kibana instance
* Increase the created rule `Additional look-back time` and restart the
rule (that will generate more alerts)
* Open the alerts page (make sure your local storage is empty otherwise,
you won't see the new column names)
### Know issues with the current implementation
* The new columns might not be displayed to users after an upgrade
* We couldn't rename `calculated_score_norm` and `calculated_score_norm`
because they are not shown by default
### Context
#### Known Alert columns issues:
_The columns are stored on local storage when the alerts page loads_
* After users open the Alerts page once, they will always see the stored
columns
* We need to create a migration when adding or updating a default
column.
* The risk score columns are displayed for users that have the risk
engine disabled.
#### Risk fields friendly names limitations
* We can only rename columns that are displayed by default.
* If the user has loaded the page, he will only see the new column name
if we write a migration to rename the columns inside local storage.
This [issue](https://github.com/elastic/kibana/issues/176125) tracks the
@elastic/response-ops team's planned improvements to column names and
visibility.
### Checklist
Delete any items that are not applicable to this PR.
- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
---
.../alerts_table/default_config.test.tsx | 24 +++++-
.../components/alerts_table/translations.ts | 28 +++++++
.../security_solution_detections/columns.ts | 28 ++++---
.../asset_criticality_badge.tsx | 2 +-
.../asset_criticality_level.test.tsx | 35 +++++++++
.../renderers/asset_criticality_level.tsx | 60 ++++++++++++++
.../body/renderers/formatted_field.tsx | 17 ++++
.../containers/local_storage/index.tsx | 2 +
.../migrates_risk_level_title.test.tsx | 78 +++++++++++++++++++
.../migrates_risk_level_title.tsx | 56 +++++++++++++
.../e2e/entity_analytics/enrichments.cy.ts | 8 +-
11 files changed, 317 insertions(+), 21 deletions(-)
create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.test.tsx
create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.tsx
create mode 100644 x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.test.tsx
create mode 100644 x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.tsx
diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx
index 1e0c7021929c9..05de5e9725399 100644
--- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx
+++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.test.tsx
@@ -67,11 +67,27 @@ const platinumBaseColumns = [
initialWidth: 450,
},
{ columnHeaderType: 'not-filtered', id: 'host.name' },
- { columnHeaderType: 'not-filtered', id: 'host.risk.calculated_level' },
{ columnHeaderType: 'not-filtered', id: 'user.name' },
- { columnHeaderType: 'not-filtered', id: 'user.risk.calculated_level' },
- { columnHeaderType: 'not-filtered', id: 'host.asset.criticality' },
- { columnHeaderType: 'not-filtered', id: 'user.asset.criticality' },
+ {
+ columnHeaderType: 'not-filtered',
+ id: 'host.risk.calculated_level',
+ displayAsText: 'Host Risk Level',
+ },
+ {
+ columnHeaderType: 'not-filtered',
+ id: 'user.risk.calculated_level',
+ displayAsText: 'User Risk Level',
+ },
+ {
+ columnHeaderType: 'not-filtered',
+ id: 'host.asset.criticality',
+ displayAsText: 'Host Criticality',
+ },
+ {
+ columnHeaderType: 'not-filtered',
+ id: 'user.asset.criticality',
+ displayAsText: 'User Criticality',
+ },
{ columnHeaderType: 'not-filtered', id: 'process.name' },
{ columnHeaderType: 'not-filtered', id: 'file.name' },
{ columnHeaderType: 'not-filtered', id: 'source.ip' },
diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts
index ae0c05d2bdb05..4a7ea8e77cc92 100644
--- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts
+++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts
@@ -137,6 +137,34 @@ export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate(
}
);
+export const ALERTS_HEADERS_HOST_RISK_LEVEL = i18n.translate(
+ 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.hostRiskLevel',
+ {
+ defaultMessage: 'Host Risk Level',
+ }
+);
+
+export const ALERTS_HEADERS_USER_RISK_LEVEL = i18n.translate(
+ 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.userRiskLevel',
+ {
+ defaultMessage: 'User Risk Level',
+ }
+);
+
+export const ALERTS_HEADERS_HOST_CRITICALITY = i18n.translate(
+ 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.hostCriticality',
+ {
+ defaultMessage: 'Host Criticality',
+ }
+);
+
+export const ALERTS_HEADERS_USER_CRITICALITY = i18n.translate(
+ 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.userCriticality',
+ {
+ defaultMessage: 'User Criticality',
+ }
+);
+
export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate(
'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle',
{
diff --git a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts
index fc3f5afa897a2..6ca0a67244179 100644
--- a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts
+++ b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts
@@ -34,6 +34,18 @@ export const assigneesColumn: ColumnHeaderOptions = {
initialWidth: DEFAULT_DATE_COLUMN_MIN_WIDTH,
};
+export const hostRiskLevelColumn: ColumnHeaderOptions = {
+ columnHeaderType: defaultColumnHeaderType,
+ id: ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL,
+ displayAsText: i18n.ALERTS_HEADERS_HOST_RISK_LEVEL,
+};
+
+export const userRiskLevelColumn: ColumnHeaderOptions = {
+ columnHeaderType: defaultColumnHeaderType,
+ id: ALERT_USER_RISK_SCORE_CALCULATED_LEVEL,
+ displayAsText: i18n.ALERTS_HEADERS_USER_RISK_LEVEL,
+};
+
const getBaseColumns = (
license?: LicenseService
): Array<
@@ -63,32 +75,24 @@ const getBaseColumns = (
columnHeaderType: defaultColumnHeaderType,
id: 'host.name',
},
- isPlatinumPlus
- ? {
- columnHeaderType: defaultColumnHeaderType,
- id: ALERT_HOST_RISK_SCORE_CALCULATED_LEVEL,
- }
- : null,
{
columnHeaderType: defaultColumnHeaderType,
id: 'user.name',
},
- isPlatinumPlus
- ? {
- columnHeaderType: defaultColumnHeaderType,
- id: ALERT_USER_RISK_SCORE_CALCULATED_LEVEL,
- }
- : null,
+ isPlatinumPlus ? hostRiskLevelColumn : null,
+ isPlatinumPlus ? userRiskLevelColumn : null,
isPlatinumPlus
? {
columnHeaderType: defaultColumnHeaderType,
id: ALERT_HOST_CRITICALITY,
+ displayAsText: i18n.ALERTS_HEADERS_HOST_CRITICALITY,
}
: null,
isPlatinumPlus
? {
columnHeaderType: defaultColumnHeaderType,
id: ALERT_USER_CRITICALITY,
+ displayAsText: i18n.ALERTS_HEADERS_USER_CRITICALITY,
}
: null,
{
diff --git a/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx b/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx
index 003d693a0182e..2fa32b82b8767 100644
--- a/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx
+++ b/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality/asset_criticality_badge.tsx
@@ -11,7 +11,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
import { CRITICALITY_LEVEL_TITLE, CRITICALITY_LEVEL_DESCRIPTION } from './translations';
import type { CriticalityLevel } from '../../../../common/entity_analytics/asset_criticality/types';
-const CRITICALITY_LEVEL_COLOR: Record = {
+export const CRITICALITY_LEVEL_COLOR: Record = {
very_important: '#E7664C',
important: '#D6BF57',
normal: '#54B399',
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.test.tsx
new file mode 100644
index 0000000000000..2a52c4509492a
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.test.tsx
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { TestProviders } from '../../../../../common/mock';
+import { render } from '@testing-library/react';
+import React from 'react';
+import { AssetCriticalityLevel } from './asset_criticality_level';
+
+jest.mock('../../../../../common/components/draggables', () => ({
+ DefaultDraggable: ({ children }: { children: React.ReactNode }) => <>{children},
+}));
+
+const defaultProps = {
+ contextId: 'testContext',
+ eventId: 'testEvent',
+ fieldName: 'testField',
+ fieldType: 'testType',
+ isAggregatable: true,
+ isDraggable: true,
+ value: 'low',
+};
+
+describe('AssetCriticalityLevel', () => {
+ it('renders', () => {
+ const { getByTestId } = render(, {
+ wrapper: TestProviders,
+ });
+
+ expect(getByTestId('AssetCriticalityLevel-score-badge')).toHaveTextContent('low');
+ });
+});
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.tsx
new file mode 100644
index 0000000000000..ac3fa8d977b61
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/asset_criticality_level.tsx
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiBadge } from '@elastic/eui';
+import { isString } from 'lodash/fp';
+import type { CriticalityLevel } from '../../../../../../common/entity_analytics/asset_criticality/types';
+import { CRITICALITY_LEVEL_COLOR } from '../../../../../entity_analytics/components/asset_criticality';
+import { DefaultDraggable } from '../../../../../common/components/draggables';
+
+interface Props {
+ contextId: string;
+ eventId: string;
+ fieldName: string;
+ fieldType: string;
+ isAggregatable: boolean;
+ isDraggable: boolean;
+ value: string | number | undefined | null;
+}
+
+const AssetCriticalityLevelComponent: React.FC = ({
+ contextId,
+ eventId,
+ fieldName,
+ fieldType,
+ isAggregatable,
+ isDraggable,
+ value,
+}) => {
+ const color = isString(value) ? CRITICALITY_LEVEL_COLOR[value as CriticalityLevel] : 'normal';
+
+ const badge = (
+
+ {value}
+
+ );
+
+ return isDraggable ? (
+
+ {badge}
+
+ ) : (
+ badge
+ );
+};
+
+export const AssetCriticalityLevel = React.memo(AssetCriticalityLevelComponent);
+AssetCriticalityLevel.displayName = 'AssetCriticalityLevel';
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx
index 7062fc7afbb78..040e6335eb8a1 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx
@@ -13,6 +13,10 @@ import { isNumber, isEmpty } from 'lodash/fp';
import React from 'react';
import { css } from '@emotion/css';
+import {
+ ALERT_HOST_CRITICALITY,
+ ALERT_USER_CRITICALITY,
+} from '../../../../../../common/field_maps/field_names';
import { SENTINEL_ONE_AGENT_ID_FIELD } from '../../../../../common/utils/sentinelone_alert_check';
import { SentinelOneAgentStatus } from '../../../../../detections/components/host_isolation/sentinel_one_agent_status';
import { EndpointAgentStatusById } from '../../../../../common/components/endpoint/endpoint_agent_status';
@@ -45,6 +49,7 @@ import { RenderRuleName, renderEventModule, renderUrl } from './formatted_field_
import { RuleStatus } from './rule_status';
import { HostName } from './host_name';
import { UserName } from './user_name';
+import { AssetCriticalityLevel } from './asset_criticality_level';
// simple black-list to prevent dragging and dropping fields such as message name
const columnNamesNotDraggable = [MESSAGE_FIELD_NAME];
@@ -256,6 +261,18 @@ const FormattedFieldValueComponent: React.FC<{
iconSide={isButton ? 'right' : undefined}
/>
);
+ } else if (fieldName === ALERT_HOST_CRITICALITY || fieldName === ALERT_USER_CRITICALITY) {
+ return (
+
+ );
} else if (fieldName === AGENT_STATUS_FIELD_NAME) {
return (
{
const tableModel = allDataTables[tableId];
diff --git a/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.test.tsx b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.test.tsx
new file mode 100644
index 0000000000000..a43e0a860e7ac
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.test.tsx
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+ LOCAL_STORAGE_MIGRATION_KEY,
+ migrateEntityRiskLevelColumnTitle,
+} from './migrates_risk_level_title';
+import type { DataTableState } from '@kbn/securitysolution-data-table';
+import {
+ hostRiskLevelColumn,
+ userRiskLevelColumn,
+} from '../../../detections/configurations/security_solution_detections/columns';
+import { Storage } from '@kbn/kibana-utils-plugin/public';
+import { localStorageMock } from '../../../common/mock/mock_local_storage';
+import { LOCAL_STORAGE_TABLE_KEY } from '.';
+
+const getColumnsBeforeMigration = () => [
+ { ...userRiskLevelColumn, displayAsText: undefined },
+ { ...hostRiskLevelColumn, displayAsText: undefined },
+];
+
+let storage: Storage;
+
+describe('migrateEntityRiskLevelColumnTitle', () => {
+ beforeEach(() => {
+ storage = new Storage(localStorageMock());
+ });
+
+ it('does NOT migrate `columns` when `columns` is not an array', () => {
+ const dataTableState = {
+ 'alerts-page': {},
+ } as unknown as DataTableState['dataTable']['tableById'];
+
+ migrateEntityRiskLevelColumnTitle(storage, dataTableState);
+
+ expect(dataTableState['alerts-page'].columns).toStrictEqual(undefined);
+ });
+
+ it('does not migrates columns if if it has already run once', () => {
+ storage.set(LOCAL_STORAGE_MIGRATION_KEY, true);
+ const dataTableState = {
+ 'alerts-page': {
+ columns: getColumnsBeforeMigration(),
+ },
+ } as unknown as DataTableState['dataTable']['tableById'];
+
+ migrateEntityRiskLevelColumnTitle(storage, dataTableState);
+
+ expect(dataTableState['alerts-page'].columns).toStrictEqual(getColumnsBeforeMigration());
+ });
+
+ it('migrates columns saved to localStorage', () => {
+ const dataTableState = {
+ 'alerts-page': {
+ columns: getColumnsBeforeMigration(),
+ },
+ } as unknown as DataTableState['dataTable']['tableById'];
+
+ migrateEntityRiskLevelColumnTitle(storage, dataTableState);
+
+ // assert that it mutates the table model
+ expect(dataTableState['alerts-page'].columns).toStrictEqual([
+ userRiskLevelColumn,
+ hostRiskLevelColumn,
+ ]);
+ // assert that it updates the migration flag on storage
+ expect(storage.get(LOCAL_STORAGE_MIGRATION_KEY)).toEqual(true);
+ // assert that it updates the table inside local storage
+ expect(storage.get(LOCAL_STORAGE_TABLE_KEY)['alerts-page'].columns).toStrictEqual([
+ userRiskLevelColumn,
+ hostRiskLevelColumn,
+ ]);
+ });
+});
diff --git a/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.tsx b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.tsx
new file mode 100644
index 0000000000000..8cbae007b3a1f
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/timelines/containers/local_storage/migrates_risk_level_title.tsx
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataTableState, TableId } from '@kbn/securitysolution-data-table';
+import { tableEntity, TableEntityType } from '@kbn/securitysolution-data-table';
+import type { Storage } from '@kbn/kibana-utils-plugin/public';
+import { LOCAL_STORAGE_TABLE_KEY } from '.';
+import {
+ hostRiskLevelColumn,
+ userRiskLevelColumn,
+} from '../../../detections/configurations/security_solution_detections/columns';
+
+export const LOCAL_STORAGE_MIGRATION_KEY =
+ 'securitySolution.dataTable.entityRiskLevelColumnTitleMigration';
+
+export const migrateEntityRiskLevelColumnTitle = (
+ storage: Storage,
+ dataTableState: DataTableState['dataTable']['tableById']
+) => {
+ // Set/Get a flag to prevent migration from running more than once
+ const hasAlreadyMigrated: boolean = storage.get(LOCAL_STORAGE_MIGRATION_KEY);
+ if (hasAlreadyMigrated) {
+ return;
+ }
+ storage.set(LOCAL_STORAGE_MIGRATION_KEY, true);
+
+ let updatedTableModel = false;
+
+ for (const [tableId, tableModel] of Object.entries(dataTableState)) {
+ // Only updates the title for alerts tables
+ if (tableEntity[tableId as TableId] === TableEntityType.alert) {
+ // In order to show correct column title after user upgrades to 8.13 we need update the stored table model with the new title.
+ const columns = tableModel.columns;
+ if (Array.isArray(columns)) {
+ columns.forEach((col) => {
+ if (col.id === userRiskLevelColumn.id) {
+ col.displayAsText = userRiskLevelColumn.displayAsText;
+ updatedTableModel = true;
+ }
+
+ if (col.id === hostRiskLevelColumn.id) {
+ col.displayAsText = hostRiskLevelColumn.displayAsText;
+ updatedTableModel = true;
+ }
+ });
+ }
+ }
+ }
+ if (updatedTableModel) {
+ storage.set(LOCAL_STORAGE_TABLE_KEY, dataTableState);
+ }
+};
diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts
index acb38b4dcefff..a080c4494833f 100644
--- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts
+++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts
@@ -63,8 +63,8 @@ describe('Enrichment', { tags: ['@ess', '@serverless'] }, () => {
});
it('Should has enrichment fields from legacy risk', function () {
- cy.get(HOST_RISK_HEADER_COLUMN).contains('host.risk.calculated_level');
- cy.get(USER_RISK_HEADER_COLUMN).contains('user.risk.calculated_level');
+ cy.get(HOST_RISK_HEADER_COLUMN).contains('Host Risk Level');
+ cy.get(USER_RISK_HEADER_COLUMN).contains('User Risk Level');
scrollAlertTableColumnIntoView(HOST_RISK_COLUMN);
cy.get(HOST_RISK_COLUMN).contains('Low');
scrollAlertTableColumnIntoView(USER_RISK_COLUMN);
@@ -103,8 +103,8 @@ describe('Enrichment', { tags: ['@ess', '@serverless'] }, () => {
});
it('Should has enrichment fields from legacy risk', function () {
- cy.get(HOST_RISK_HEADER_COLUMN).contains('host.risk.calculated_level');
- cy.get(USER_RISK_HEADER_COLUMN).contains('user.risk.calculated_level');
+ cy.get(HOST_RISK_HEADER_COLUMN).contains('Host Risk Level');
+ cy.get(USER_RISK_HEADER_COLUMN).contains('User Risk Level');
scrollAlertTableColumnIntoView(HOST_RISK_COLUMN);
cy.get(HOST_RISK_COLUMN).contains('Critical');
scrollAlertTableColumnIntoView(USER_RISK_COLUMN);
From bc19b6b13c2c0c8d461586a17de7b923d319df18 Mon Sep 17 00:00:00 2001
From: Alex Szabo
Date: Fri, 9 Feb 2024 10:31:57 +0100
Subject: [PATCH 08/13] [Ops] Increase step timeout for storybook build
(#176501)
## Summary
Several instances of post-merge build failed on the storybook build and
upload, as it just finished briefly within limit, the step altogether
timed out.
This PR increases the timeout by 20m (a generous increment) while taking
note on ideally speeding up storybook builds:
https://github.com/elastic/kibana/issues/176500
---
.buildkite/pipelines/on_merge.yml | 2 +-
.buildkite/pipelines/pull_request/storybooks.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
index 2841dce49bb26..fcf4a82c0801c 100644
--- a/.buildkite/pipelines/on_merge.yml
+++ b/.buildkite/pipelines/on_merge.yml
@@ -372,7 +372,7 @@ steps:
agents:
queue: n2-8-spot
key: storybooks
- timeout_in_minutes: 60
+ timeout_in_minutes: 80
retry:
automatic:
- exit_status: '-1'
diff --git a/.buildkite/pipelines/pull_request/storybooks.yml b/.buildkite/pipelines/pull_request/storybooks.yml
index 81d13b628e049..8f76879231de2 100644
--- a/.buildkite/pipelines/pull_request/storybooks.yml
+++ b/.buildkite/pipelines/pull_request/storybooks.yml
@@ -4,4 +4,4 @@ steps:
agents:
queue: n2-8-spot
key: storybooks
- timeout_in_minutes: 60
+ timeout_in_minutes: 80
From 240e5ef10c6f5763724510b727ce07c82fe15498 Mon Sep 17 00:00:00 2001
From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>
Date: Fri, 9 Feb 2024 10:39:57 +0100
Subject: [PATCH 09/13] [Fleet] Make datastream rollover lazy (#174790)
(#176565)
## Summary
Add back changes in https://github.com/elastic/kibana/pull/174790 after
https://github.com/elastic/elasticsearch/issues/104732 is fixed
Resolve https://github.com/elastic/kibana/issues/174480
Co-authored-by: Nicolas Chaulet
---
.../elasticsearch/template/template.test.ts | 9 +-
.../epm/elasticsearch/template/template.ts | 8 +-
.../apis/epm/data_stream.ts | 96 +++++++++++--------
.../apis/epm/install_hidden_datastreams.ts | 80 +++++++++-------
4 files changed, 113 insertions(+), 80 deletions(-)
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
index f680a0bf004a6..58bcfcca386cf 100644
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
@@ -1673,7 +1673,14 @@ describe('EPM template', () => {
},
]);
- expect(esClient.indices.rollover).toHaveBeenCalled();
+ expect(esClient.transport.request).toHaveBeenCalledWith(
+ expect.objectContaining({
+ path: '/test.prefix1-default/_rollover',
+ querystring: {
+ lazy: true,
+ },
+ })
+ );
});
it('should skip rollover on expected error when flag is on', async () => {
const esClient = elasticsearchServiceMock.createElasticsearchClient();
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
index da2b801548e18..01b1792dc5e79 100644
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
@@ -946,8 +946,12 @@ const getDataStreams = async (
const rolloverDataStream = (dataStreamName: string, esClient: ElasticsearchClient) => {
try {
// Do no wrap rollovers in retryTransientEsErrors since it is not idempotent
- return esClient.indices.rollover({
- alias: dataStreamName,
+ return esClient.transport.request({
+ method: 'POST',
+ path: `/${dataStreamName}/_rollover`,
+ querystring: {
+ lazy: true,
+ },
});
} catch (error) {
throw new PackageESError(
diff --git a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts
index 06a67a13e425c..a257ff97933d9 100644
--- a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts
+++ b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts
@@ -41,43 +41,46 @@ export default function (providerContext: FtrProviderContext) {
skipIfNoDockerRegistry(providerContext);
setupFleetAndAgents(providerContext);
+ const writeMetricsDoc = (namespace: string) =>
+ es.transport.request(
+ {
+ method: 'POST',
+ path: `/${metricsTemplateName}-${namespace}/_doc?refresh=true`,
+ body: {
+ '@timestamp': new Date().toISOString(),
+ logs_test_name: 'test',
+ data_stream: {
+ dataset: `${pkgName}.test_metrics`,
+ namespace,
+ type: 'metrics',
+ },
+ },
+ },
+ { meta: true }
+ );
+
+ const writeLogsDoc = (namespace: string) =>
+ es.transport.request(
+ {
+ method: 'POST',
+ path: `/${logsTemplateName}-${namespace}/_doc?refresh=true`,
+ body: {
+ '@timestamp': new Date().toISOString(),
+ logs_test_name: 'test',
+ data_stream: {
+ dataset: `${pkgName}.test_logs`,
+ namespace,
+ type: 'logs',
+ },
+ },
+ },
+ { meta: true }
+ );
beforeEach(async () => {
await installPackage(pkgName, pkgVersion);
await Promise.all(
namespaces.map(async (namespace) => {
- const createLogsRequest = es.transport.request(
- {
- method: 'POST',
- path: `/${logsTemplateName}-${namespace}/_doc`,
- body: {
- '@timestamp': '2015-01-01',
- logs_test_name: 'test',
- data_stream: {
- dataset: `${pkgName}.test_logs`,
- namespace,
- type: 'logs',
- },
- },
- },
- { meta: true }
- );
- const createMetricsRequest = es.transport.request(
- {
- method: 'POST',
- path: `/${metricsTemplateName}-${namespace}/_doc`,
- body: {
- '@timestamp': '2015-01-01',
- logs_test_name: 'test',
- data_stream: {
- dataset: `${pkgName}.test_metrics`,
- namespace,
- type: 'metrics',
- },
- },
- },
- { meta: true }
- );
- return Promise.all([createLogsRequest, createMetricsRequest]);
+ return Promise.all([writeLogsDoc(namespace), writeMetricsDoc(namespace)]);
})
);
});
@@ -141,7 +144,11 @@ export default function (providerContext: FtrProviderContext) {
it('after update, it should have rolled over logs datastream because mappings are not compatible and not metrics', async function () {
await installPackage(pkgName, pkgUpdateVersion);
+
await asyncForEach(namespaces, async (namespace) => {
+ // write doc as rollover is lazy
+ await writeLogsDoc(namespace);
+ await writeMetricsDoc(namespace);
const resLogsDatastream = await es.transport.request(
{
method: 'GET',
@@ -266,6 +273,8 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(200);
+ // Write a doc to trigger lazy rollover
+ await writeLogsDoc('default');
// Datastream should have been rolled over
expect(await getLogsDefaultBackingIndicesLength()).to.be(2);
});
@@ -303,26 +312,29 @@ export default function (providerContext: FtrProviderContext) {
skipIfNoDockerRegistry(providerContext);
setupFleetAndAgents(providerContext);
- beforeEach(async () => {
- await installPackage(pkgName, pkgVersion);
-
- // Create a sample document so the data stream is created
- await es.transport.request(
+ const writeMetricDoc = (body: any = {}) =>
+ es.transport.request(
{
method: 'POST',
- path: `/${metricsTemplateName}-${namespace}/_doc`,
+ path: `/${metricsTemplateName}-${namespace}/_doc?refresh=true`,
body: {
- '@timestamp': '2015-01-01',
+ '@timestamp': new Date().toISOString(),
logs_test_name: 'test',
data_stream: {
dataset: `${pkgName}.test_logs`,
namespace,
type: 'logs',
},
+ ...body,
},
},
{ meta: true }
);
+ beforeEach(async () => {
+ await installPackage(pkgName, pkgVersion);
+
+ // Create a sample document so the data stream is created
+ await writeMetricDoc();
});
afterEach(async () => {
@@ -340,6 +352,10 @@ export default function (providerContext: FtrProviderContext) {
it('rolls over data stream when index_mode: time_series is set in the updated package version', async () => {
await installPackage(pkgName, pkgUpdateVersion);
+ // Write a doc so lazy rollover can happen
+ await writeMetricDoc({
+ some_field: 'test',
+ });
const resMetricsDatastream = await es.transport.request(
{
method: 'GET',
diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts
index 2fe976352944a..2ec6fb92000e3 100644
--- a/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts
+++ b/x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts
@@ -34,46 +34,50 @@ export default function (providerContext: FtrProviderContext) {
.send({ force: true })
.expect(200);
- await es.index({
- index: 'metrics-apm.service_summary.10m-default',
- document: {
- '@timestamp': '2023-05-30T07:50:00.000Z',
- agent: {
- name: 'go',
- },
- data_stream: {
- dataset: 'apm.service_summary.10m',
- namespace: 'default',
- type: 'metrics',
- },
- ecs: {
- version: '8.6.0-dev',
- },
- event: {
- agent_id_status: 'missing',
- ingested: '2023-05-30T07:57:12Z',
- },
- metricset: {
- interval: '10m',
- name: 'service_summary',
- },
- observer: {
- hostname: '047e282994fb',
- type: 'apm-server',
- version: '8.7.0',
- },
- processor: {
- event: 'metric',
- name: 'metric',
- },
- service: {
- language: {
+ const writeDoc = () =>
+ es.index({
+ refresh: true,
+ index: 'metrics-apm.service_summary.10m-default',
+ document: {
+ '@timestamp': '2023-05-30T07:50:00.000Z',
+ agent: {
name: 'go',
},
- name: '___main_elastic_cloud_87_ilm_fix',
+ data_stream: {
+ dataset: 'apm.service_summary.10m',
+ namespace: 'default',
+ type: 'metrics',
+ },
+ ecs: {
+ version: '8.6.0-dev',
+ },
+ event: {
+ agent_id_status: 'missing',
+ ingested: '2023-05-30T07:57:12Z',
+ },
+ metricset: {
+ interval: '10m',
+ name: 'service_summary',
+ },
+ observer: {
+ hostname: '047e282994fb',
+ type: 'apm-server',
+ version: '8.7.0',
+ },
+ processor: {
+ event: 'metric',
+ name: 'metric',
+ },
+ service: {
+ language: {
+ name: 'go',
+ },
+ name: '___main_elastic_cloud_87_ilm_fix',
+ },
},
- },
- });
+ });
+
+ await writeDoc();
await supertest
.post(`/api/fleet/epm/packages/apm/8.8.0`)
@@ -81,6 +85,8 @@ export default function (providerContext: FtrProviderContext) {
.send({ force: true })
.expect(200);
+ // Rollover are lazy need to write a new doc
+ await writeDoc();
const ds = await es.indices.get({
index: 'metrics-apm.service_summary*',
expand_wildcards: ['open', 'hidden'],
From cd1f2b02bb81a94af5fea0e63b79f36f249a8e16 Mon Sep 17 00:00:00 2001
From: jennypavlova
Date: Fri, 9 Feb 2024 11:06:14 +0100
Subject: [PATCH 10/13] [ObsUx][Infra] Add collapsible sections in the overview
tab (#175716)
Closes #175989
## Summary
This PR is a follow-up to
https://github.com/elastic/kibana/issues/175558. It adds the active
alerts count next to the alert section title (this will happen after the
alerts widget is loaded) following the rules:
Default behaviour
No alerts at all ==> Collapse and say 'No active alerts'
No active alerts ==> Collapse and say 'No active alerts'
Active alerts ==> Expand fully
Collapsed
No alerts at all ==> Say 'No active alerts'
No active alerts ==> Say 'No active alerts'
Active alerts ==> say "X active alerts"
It adds a change in the `AlertSummaryWidget` to make it possible to get
the alerts count after the widget is loaded using a new prop.
This PR also changes the alerts tab active alert count badge color on
the hosts view to keep it consistent:
| Before | After |
| ------ | ------ |
| 
| 
|
## Testing
- Open hosts view and select a host with active alerts (flyout or full
page)
- The alerts section should be expanded showing the alerts widget
fd1a21035a5f)
- Collapse the alerts section by clicking on the title or the button:
- Open hosts view and select a host without active alerts (flyout or
full page)
- The alerts section should be collapsed showing the message 'No active
alerts'
https://github.com/elastic/kibana/assets/14139027/4058ed69-95f5-4b4c-8925-6680ac3791c1
---
.../asset_details/tabs/overview/alerts.tsx | 30 ++++++++++---
.../tabs/overview/alerts_closed_content.tsx | 44 ++++++++++++++++++
.../overview/section/collapsible_section.tsx | 11 ++++-
.../infra/public/hooks/use_alerts_count.ts | 2 +-
.../components/tabs/alerts_tab_badge.tsx | 8 ++--
.../alert_summary_widget.tsx | 4 +-
.../sections/alert_summary_widget/types.ts | 7 ++-
.../functional/apps/infra/node_details.ts | 45 +++++++++++++++++++
.../functional/page_objects/asset_details.ts | 18 ++++++++
9 files changed, 154 insertions(+), 15 deletions(-)
create mode 100644 x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts_closed_content.tsx
diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx
index 1fc8cc6614a75..4b7a1907e206a 100644
--- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx
+++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts.tsx
@@ -4,9 +4,9 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
-import React, { useMemo } from 'react';
+import React, { useMemo, useState } from 'react';
-import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import { EuiFlexGroup, EuiFlexItem, type EuiAccordionProps } from '@elastic/eui';
import { useSummaryTimeRange } from '@kbn/observability-plugin/public';
import type { TimeRange } from '@kbn/es-query';
import type { InventoryItemType } from '@kbn/metrics-data-access-plugin/common';
@@ -24,6 +24,8 @@ import { ALERT_STATUS_ALL } from '../../../../common/alerts/constants';
import { AlertsSectionTitle } from '../../components/section_titles';
import { useAssetDetailsRenderPropsContext } from '../../hooks/use_asset_details_render_props';
import { CollapsibleSection } from './section/collapsible_section';
+import { AlertsClosedContent } from './alerts_closed_content';
+import { type AlertsCount } from '../../../../hooks/use_alerts_count';
export const AlertsSummaryContent = ({
assetName,
@@ -37,6 +39,9 @@ export const AlertsSummaryContent = ({
const { featureFlags } = usePluginConfig();
const [isAlertFlyoutVisible, { toggle: toggleAlertFlyout }] = useBoolean(false);
const { overrides } = useAssetDetailsRenderPropsContext();
+ const [collapsibleStatus, setCollapsibleStatus] =
+ useState('open');
+ const [activeAlertsCount, setActiveAlertsCount] = useState(undefined);
const alertsEsQueryByStatus = useMemo(
() =>
@@ -48,6 +53,14 @@ export const AlertsSummaryContent = ({
[assetName, dateRange]
);
+ const onLoaded = (alertsCount?: AlertsCount) => {
+ const { activeAlertCount = 0 } = alertsCount ?? {};
+ const hasActiveAlerts = activeAlertCount > 0;
+
+ setCollapsibleStatus(hasActiveAlerts ? 'open' : 'closed');
+ setActiveAlertsCount(alertsCount?.activeAlertCount);
+ };
+
return (
<>
}
+ initialTriggerValue={collapsibleStatus}
extraAction={
{featureFlags.inventoryThresholdAlertRuleEnabled && (
@@ -72,9 +87,12 @@ export const AlertsSummaryContent = ({
}
>
-
+
-
{featureFlags.inventoryThresholdAlertRuleEnabled && (
void;
}
const MemoAlertSummaryWidget = React.memo(
- ({ alertsQuery, dateRange }: MemoAlertSummaryWidgetProps) => {
+ ({ alertsQuery, dateRange, onLoaded }: MemoAlertSummaryWidgetProps) => {
const { services } = useKibanaContextForPlugin();
const summaryTimeRange = useSummaryTimeRange(dateRange);
@@ -112,6 +131,7 @@ const MemoAlertSummaryWidget = React.memo(
featureIds={infraAlertFeatureIds}
filter={alertsQuery}
timeRange={summaryTimeRange}
+ onLoaded={onLoaded}
fullSize
hideChart
/>
diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts_closed_content.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts_closed_content.tsx
new file mode 100644
index 0000000000000..a08a0313230ee
--- /dev/null
+++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/alerts_closed_content.tsx
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { EuiBadge, EuiToolTip } from '@elastic/eui';
+
+export const AlertsClosedContent = ({ activeAlertCount }: { activeAlertCount?: number }) => {
+ const shouldRenderAlertsClosedContent = typeof activeAlertCount === 'number';
+
+ if (!shouldRenderAlertsClosedContent) {
+ return null;
+ }
+
+ if (activeAlertCount > 0) {
+ return (
+
+
+ {activeAlertCount}
+
+
+ );
+ }
+
+ return (
+
+ {i18n.translate('xpack.infra.assetDetails.noActiveAlertsContentClosedSection', {
+ defaultMessage: 'No active alerts',
+ })}
+
+ );
+};
diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx
index ec31851d89a6d..da0b993199ee1 100644
--- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx
+++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/section/collapsible_section.tsx
@@ -5,6 +5,8 @@
* 2.0.
*/
+import React, { useEffect, useState } from 'react';
+
import {
EuiAccordion,
EuiFlexGroup,
@@ -12,7 +14,6 @@ import {
useGeneratedHtmlId,
type EuiAccordionProps,
} from '@elastic/eui';
-import React, { useState } from 'react';
export const CollapsibleSection = ({
title,
@@ -22,6 +23,7 @@ export const CollapsibleSection = ({
collapsible,
['data-test-subj']: dataTestSubj,
id,
+ initialTriggerValue,
}: {
title: React.FunctionComponent;
closedSectionContent?: React.ReactNode;
@@ -31,13 +33,18 @@ export const CollapsibleSection = ({
collapsible: boolean;
['data-test-subj']: string;
id: string;
+ initialTriggerValue?: EuiAccordionProps['forceState'];
}) => {
const [trigger, setTrigger] = useState('open');
+ useEffect(() => {
+ setTrigger(initialTriggerValue ?? 'open');
+ }, [initialTriggerValue]);
+
const Title = title;
const ButtonContent = () =>
closedSectionContent && trigger === 'closed' ? (
-
+
diff --git a/x-pack/plugins/infra/public/hooks/use_alerts_count.ts b/x-pack/plugins/infra/public/hooks/use_alerts_count.ts
index 7d05a275d6eae..5c602d09b7d23 100644
--- a/x-pack/plugins/infra/public/hooks/use_alerts_count.ts
+++ b/x-pack/plugins/infra/public/hooks/use_alerts_count.ts
@@ -28,7 +28,7 @@ interface FetchAlertsCountParams {
signal: AbortSignal;
}
-interface AlertsCount {
+export interface AlertsCount {
activeAlertCount: number;
recoveredAlertCount: number;
}
diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx
index 15c4e568ad8ed..c3f778299ab69 100644
--- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx
+++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/alerts_tab_badge.tsx
@@ -5,7 +5,7 @@
* 2.0.
*/
import React from 'react';
-import { EuiIcon, EuiLoadingSpinner, EuiNotificationBadge, EuiToolTip } from '@elastic/eui';
+import { EuiIcon, EuiLoadingSpinner, EuiBadge, EuiToolTip } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { useAlertsCount } from '../../../../../hooks/use_alerts_count';
import { infraAlertFeatureIds } from './config';
@@ -40,12 +40,12 @@ export const AlertsTabBadge = () => {
typeof alertsCount?.activeAlertCount === 'number' && alertsCount.activeAlertCount > 0;
return shouldRenderBadge ? (
-
{alertsCount?.activeAlertCount}
-
+
) : null;
};
diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx
index 2031c9a1f3fe2..2619ef2b25258 100644
--- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx
+++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/alert_summary_widget.tsx
@@ -37,9 +37,9 @@ export const AlertSummaryWidget = ({
useEffect(() => {
if (!isLoading && onLoaded) {
- onLoaded();
+ onLoaded({ activeAlertCount, recoveredAlertCount });
}
- }, [isLoading, onLoaded]);
+ }, [activeAlertCount, isLoading, onLoaded, recoveredAlertCount]);
if (isLoading)
return ;
diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts
index 48a49acf5ad7c..1bb02adff0653 100644
--- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts
+++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_summary_widget/types.ts
@@ -30,6 +30,11 @@ export interface ChartProps {
onBrushEnd?: BrushEndListener;
}
+interface AlertsCount {
+ activeAlertCount: number;
+ recoveredAlertCount: number;
+}
+
export interface AlertSummaryWidgetProps {
featureIds?: ValidFeatureId[];
filter?: estypes.QueryDslQueryContainer;
@@ -38,5 +43,5 @@ export interface AlertSummaryWidgetProps {
timeRange: AlertSummaryTimeRange;
chartProps: ChartProps;
hideChart?: boolean;
- onLoaded?: () => void;
+ onLoaded?: (alertsCount?: AlertsCount) => void;
}
diff --git a/x-pack/test/functional/apps/infra/node_details.ts b/x-pack/test/functional/apps/infra/node_details.ts
index 172ac410eb427..42147652f34a5 100644
--- a/x-pack/test/functional/apps/infra/node_details.ts
+++ b/x-pack/test/functional/apps/infra/node_details.ts
@@ -204,6 +204,15 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
await pageObjects.assetDetails.overviewAlertsTitleExists();
});
+ it('should show / hide alerts section with no alerts and show / hide closed section content', async () => {
+ await pageObjects.assetDetails.alertsSectionCollapsibleExist();
+ // Collapsed by default
+ await pageObjects.assetDetails.alertsSectionClosedContentNoAlertsExist();
+ // Expand
+ await pageObjects.assetDetails.alertsSectionCollapsibleClick();
+ await pageObjects.assetDetails.alertsSectionClosedContentNoAlertsMissing();
+ });
+
it('shows the CPU Profiling prompt if UI setting for Profiling integration is enabled', async () => {
await setInfrastructureProfilingIntegrationUiSetting(true);
await pageObjects.assetDetails.cpuProfilingPromptExists();
@@ -213,6 +222,42 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
await setInfrastructureProfilingIntegrationUiSetting(false);
await pageObjects.assetDetails.cpuProfilingPromptMissing();
});
+
+ describe('Alerts Section with alerts', () => {
+ before(async () => {
+ await navigateToNodeDetails('demo-stack-apache-01', 'demo-stack-apache-01');
+ await pageObjects.header.waitUntilLoadingHasFinished();
+
+ await pageObjects.timePicker.setAbsoluteRange(
+ START_HOST_ALERTS_DATE.format(DATE_PICKER_FORMAT),
+ END_HOST_ALERTS_DATE.format(DATE_PICKER_FORMAT)
+ );
+
+ await pageObjects.assetDetails.clickOverviewTab();
+ });
+
+ after(async () => {
+ await navigateToNodeDetails('Jennys-MBP.fritz.box', 'Jennys-MBP.fritz.box');
+ await pageObjects.header.waitUntilLoadingHasFinished();
+
+ await pageObjects.timePicker.setAbsoluteRange(
+ START_HOST_PROCESSES_DATE.format(DATE_PICKER_FORMAT),
+ END_HOST_PROCESSES_DATE.format(DATE_PICKER_FORMAT)
+ );
+ });
+
+ it('should show / hide alerts section with active alerts and show / hide closed section content', async () => {
+ await pageObjects.assetDetails.alertsSectionCollapsibleExist();
+ // Expanded by default
+ await pageObjects.assetDetails.alertsSectionClosedContentMissing();
+ // Collapse
+ await pageObjects.assetDetails.alertsSectionCollapsibleClick();
+ await pageObjects.assetDetails.alertsSectionClosedContentExist();
+ // Expand
+ await pageObjects.assetDetails.alertsSectionCollapsibleClick();
+ await pageObjects.assetDetails.alertsSectionClosedContentMissing();
+ });
+ });
});
describe('Metadata Tab', () => {
diff --git a/x-pack/test/functional/page_objects/asset_details.ts b/x-pack/test/functional/page_objects/asset_details.ts
index 5e1ea574f8a81..cd34d9c2ca10b 100644
--- a/x-pack/test/functional/page_objects/asset_details.ts
+++ b/x-pack/test/functional/page_objects/asset_details.ts
@@ -89,6 +89,24 @@ export function AssetDetailsProvider({ getService }: FtrProviderContext) {
return await testSubjects.existOrFail('infraAssetDetailsMetricsCollapsible');
},
+ async alertsSectionCollapsibleClick() {
+ return await testSubjects.click('infraAssetDetailsAlertsCollapsible');
+ },
+
+ async alertsSectionClosedContentExist() {
+ return await testSubjects.existOrFail('infraAssetDetailsAlertsClosedContentWithAlerts');
+ },
+ async alertsSectionClosedContentMissing() {
+ return await testSubjects.missingOrFail('infraAssetDetailsAlertsClosedContentWithAlerts');
+ },
+
+ async alertsSectionClosedContentNoAlertsExist() {
+ return await testSubjects.existOrFail('infraAssetDetailsAlertsClosedContentNoAlerts');
+ },
+ async alertsSectionClosedContentNoAlertsMissing() {
+ return await testSubjects.missingOrFail('infraAssetDetailsAlertsClosedContentNoAlerts');
+ },
+
// Metadata
async clickMetadataTab() {
return testSubjects.click('infraAssetDetailsMetadataTab');
From d0fbec8e49019631f07430ef644f64b3e9513a77 Mon Sep 17 00:00:00 2001
From: James Gowdy
Date: Fri, 9 Feb 2024 10:42:58 +0000
Subject: [PATCH 11/13] [ML] Adding delay to deletion model to avoid flickering
(#176424)
Fixes https://github.com/elastic/kibana/issues/173790
We now wait 1 second before showing the model. This should hopefully
reduce the chance of briefly seeing the initial `Checking to see...`
modal.
https://github.com/elastic/kibana/assets/22172091/0d86bdfd-3e31-4b67-af6c-48b2d7681a00
---
.../delete_space_aware_item_check_modal.tsx | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx b/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx
index d436a52f7ccb3..22500f936e705 100644
--- a/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx
+++ b/x-pack/plugins/ml/public/application/components/delete_space_aware_item_check_modal/delete_space_aware_item_check_modal.tsx
@@ -21,6 +21,7 @@ import {
EuiText,
EuiSpacer,
} from '@elastic/eui';
+import useDebounce from 'react-use/lib/useDebounce';
import type {
CanDeleteMLSpaceAwareItemsResponse,
MlSavedObjectType,
@@ -246,12 +247,16 @@ export const DeleteSpaceAwareItemCheckModal: FC = ({
const [itemCheckRespSummary, setItemCheckRespSummary] = useState<
CanDeleteMLSpaceAwareItemsSummary | undefined
>();
+ const [showModal, setShowModal] = useState(false);
const {
savedObjects: { canDeleteMLSpaceAwareItems, removeItemFromCurrentSpace },
} = useMlApiContext();
const { displayErrorToast, displaySuccessToast } = useToastNotificationService();
+ // delay showing the modal to avoid flickering
+ useDebounce(() => setShowModal(true), 1000);
+
useEffect(() => {
setIsLoading(true);
// Do the spaces check and set the content for the modal and buttons depending on results
@@ -321,6 +326,10 @@ export const DeleteSpaceAwareItemCheckModal: FC = ({
}
};
+ if (showModal === false) {
+ return null;
+ }
+
return (
<>
From 3e0d73837519167473a43841ae98a68fe7850a49 Mon Sep 17 00:00:00 2001
From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com>
Date: Fri, 9 Feb 2024 11:59:19 +0100
Subject: [PATCH 12/13] [Search] Enable API key management for Native
Connectors (#176327)
### Changes
- Show the API key configuration section in Native Connector
configuration pages
- Allow `generateApiKey` to accept params for native connectors
- Create/update secret with encoded API key as appropriate
- Unit tests
---
packages/kbn-search-connectors/lib/index.ts | 1 +
.../lib/update_connector_secret.test.ts | 42 ++++
.../lib/update_connector_secret.ts | 24 +++
...nerate_connector_api_key_api_logic.test.ts | 30 ++-
.../generate_connector_api_key_api_logic.ts | 18 +-
.../connector/api_key_configuration.tsx | 47 +++--
.../connector/connector_configuration.tsx | 7 +-
.../native_connector_configuration.tsx | 23 ++
.../lib/connectors/add_connector.test.ts | 24 +--
.../server/lib/connectors/add_connector.ts | 11 +-
.../lib/indices/generate_api_key.test.ts | 199 +++++++++++++++++-
.../server/lib/indices/generate_api_key.ts | 42 +++-
.../routes/enterprise_search/indices.ts | 8 +-
13 files changed, 409 insertions(+), 67 deletions(-)
create mode 100644 packages/kbn-search-connectors/lib/update_connector_secret.test.ts
create mode 100644 packages/kbn-search-connectors/lib/update_connector_secret.ts
diff --git a/packages/kbn-search-connectors/lib/index.ts b/packages/kbn-search-connectors/lib/index.ts
index e0a1caea66422..e7269a0620b62 100644
--- a/packages/kbn-search-connectors/lib/index.ts
+++ b/packages/kbn-search-connectors/lib/index.ts
@@ -23,5 +23,6 @@ export * from './update_connector_configuration';
export * from './update_connector_index_name';
export * from './update_connector_name_and_description';
export * from './update_connector_scheduling';
+export * from './update_connector_secret';
export * from './update_connector_service_type';
export * from './update_connector_status';
diff --git a/packages/kbn-search-connectors/lib/update_connector_secret.test.ts b/packages/kbn-search-connectors/lib/update_connector_secret.test.ts
new file mode 100644
index 0000000000000..80eb98a37babd
--- /dev/null
+++ b/packages/kbn-search-connectors/lib/update_connector_secret.test.ts
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+
+import { updateConnectorSecret } from './update_connector_secret';
+
+describe('updateConnectorSecret lib function', () => {
+ const mockClient = {
+ transport: {
+ request: jest.fn(),
+ },
+ };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ jest.useFakeTimers();
+ });
+
+ it('should update a connector secret', async () => {
+ mockClient.transport.request.mockImplementation(() => ({
+ result: 'created',
+ }));
+
+ await expect(
+ updateConnectorSecret(mockClient as unknown as ElasticsearchClient, 'my-secret', 'secret-id')
+ ).resolves.toEqual({ result: 'created' });
+ expect(mockClient.transport.request).toHaveBeenCalledWith({
+ method: 'PUT',
+ path: '/_connector/_secret/secret-id',
+ body: {
+ value: 'my-secret',
+ },
+ });
+ jest.useRealTimers();
+ });
+});
diff --git a/packages/kbn-search-connectors/lib/update_connector_secret.ts b/packages/kbn-search-connectors/lib/update_connector_secret.ts
new file mode 100644
index 0000000000000..516818b7e9b8d
--- /dev/null
+++ b/packages/kbn-search-connectors/lib/update_connector_secret.ts
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { ConnectorsAPIUpdateResponse } from '../types/connectors_api';
+
+export const updateConnectorSecret = async (
+ client: ElasticsearchClient,
+ value: string,
+ secretId: string
+) => {
+ return await client.transport.request({
+ method: 'PUT',
+ path: `/_connector/_secret/${secretId}`,
+ body: {
+ value,
+ },
+ });
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts
index 524bc70e9b279..55161f5912cfe 100644
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.test.ts
@@ -11,19 +11,43 @@ import { nextTick } from '@kbn/test-jest-helpers';
import { generateApiKey } from './generate_connector_api_key_api_logic';
+jest.mock('@kbn/search-connectors', () => ({
+ createConnectorSecret: jest.fn(),
+ updateConnectorSecret: jest.fn(),
+}));
+
describe('generateConnectorApiKeyApiLogic', () => {
const { http } = mockHttpValues;
beforeEach(() => {
jest.clearAllMocks();
});
- describe('generateApiKey', () => {
+ describe('generateApiKey for connector clients', () => {
+ it('calls correct api', async () => {
+ const promise = Promise.resolve('result');
+ http.post.mockReturnValue(promise);
+ const result = generateApiKey({ indexName: 'indexName', isNative: false, secretId: null });
+ await nextTick();
+ expect(http.post).toHaveBeenCalledWith(
+ '/internal/enterprise_search/indices/indexName/api_key',
+ {
+ body: '{"is_native":false,"secret_id":null}',
+ }
+ );
+ await expect(result).resolves.toEqual('result');
+ });
+ });
+
+ describe('generateApiKey for native connectors', () => {
it('calls correct api', async () => {
const promise = Promise.resolve('result');
http.post.mockReturnValue(promise);
- const result = generateApiKey({ indexName: 'indexName' });
+ const result = generateApiKey({ indexName: 'indexName', isNative: true, secretId: '1234' });
await nextTick();
expect(http.post).toHaveBeenCalledWith(
- '/internal/enterprise_search/indices/indexName/api_key'
+ '/internal/enterprise_search/indices/indexName/api_key',
+ {
+ body: '{"is_native":true,"secret_id":"1234"}',
+ }
);
await expect(result).resolves.toEqual('result');
});
diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts
index ace963d9208be..ebca9c99add0d 100644
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/generate_connector_api_key_api_logic.ts
@@ -15,9 +15,23 @@ export interface ApiKey {
name: string;
}
-export const generateApiKey = async ({ indexName }: { indexName: string }) => {
+export const generateApiKey = async ({
+ indexName,
+ isNative,
+ secretId,
+}: {
+ indexName: string;
+ isNative: boolean;
+ secretId: string | null;
+}) => {
const route = `/internal/enterprise_search/indices/${indexName}/api_key`;
- return await HttpLogic.values.http.post(route);
+ const params = {
+ is_native: isNative,
+ secret_id: secretId,
+ };
+ return await HttpLogic.values.http.post(route, {
+ body: JSON.stringify(params),
+ });
};
export const GenerateConnectorApiKeyApiLogic = createApiLogic(
diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx
index fdc36863f4925..b2233bb4c9ee1 100644
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/api_key_configuration.tsx
@@ -61,10 +61,12 @@ const ConfirmModal: React.FC<{
);
-export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> = ({
- hasApiKey,
- indexName,
-}) => {
+export const ApiKeyConfig: React.FC<{
+ hasApiKey: boolean;
+ indexName: string;
+ isNative: boolean;
+ secretId: string | null;
+}> = ({ hasApiKey, indexName, isNative, secretId }) => {
const { makeRequest, apiReset } = useActions(GenerateConnectorApiKeyApiLogic);
const { data, status } = useValues(GenerateConnectorApiKeyApiLogic);
useEffect(() => {
@@ -76,7 +78,7 @@ export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> =
if (hasApiKey || data) {
setIsModalVisible(true);
} else {
- makeRequest({ indexName });
+ makeRequest({ indexName, isNative, secretId });
}
};
@@ -87,7 +89,7 @@ export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> =
};
const onConfirm = () => {
- makeRequest({ indexName });
+ makeRequest({ indexName, isNative, secretId });
setIsModalVisible(false);
};
@@ -96,17 +98,28 @@ export const ApiKeyConfig: React.FC<{ hasApiKey: boolean; indexName: string }> =
{isModalVisible && }
- {i18n.translate(
- 'xpack.enterpriseSearch.content.indices.configurationConnector.apiKey.description',
- {
- defaultMessage:
- 'First, generate an Elasticsearch API key. This {apiKeyName} key will enable read and write permissions for the connector to index documents to the created {indexName} index. Save the key in a safe place, as you will need it to configure your connector.',
- values: {
- apiKeyName: `${indexName}-connector`,
- indexName,
- },
- }
- )}
+ {isNative
+ ? i18n.translate(
+ 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.apiKey.description',
+ {
+ defaultMessage: `This native connector's API key {apiKeyName} is managed internally by Elasticsearch. The connector uses this API key to index documents into the {indexName} index. To rollover your API key, click "Generate API key".`,
+ values: {
+ apiKeyName: `${indexName}-connector`,
+ indexName,
+ },
+ }
+ )
+ : i18n.translate(
+ 'xpack.enterpriseSearch.content.indices.configurationConnector.apiKey.description',
+ {
+ defaultMessage:
+ 'First, generate an Elasticsearch API key. This {apiKeyName} key will enable read and write permissions for the connector to index documents to the created {indexName} index. Save the key in a safe place, as you will need it to configure your connector.',
+ values: {
+ apiKeyName: `${indexName}-connector`,
+ indexName,
+ },
+ }
+ )}
diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx
index e338b7d1f193b..748815d4421b6 100644
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration.tsx
@@ -95,7 +95,12 @@ export const ConnectorConfiguration: React.FC = () => {
steps={[
{
children: (
-
+
),
status: hasApiKey ? 'complete' : 'incomplete',
title: i18n.translate(
diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx
index fce32710a580d..3457e3c709fca 100644
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/native_connector_configuration.tsx
@@ -30,9 +30,11 @@ import { HttpLogic } from '../../../../../shared/http';
import { CONNECTOR_ICONS } from '../../../../../shared/icons/connector_icons';
import { KibanaLogic } from '../../../../../shared/kibana';
+import { GenerateConnectorApiKeyApiLogic } from '../../../../api/connector/generate_connector_api_key_api_logic';
import { hasConfiguredConfiguration } from '../../../../utils/has_configured_configuration';
import { isConnectorIndex } from '../../../../utils/indices';
import { IndexViewLogic } from '../../index_view_logic';
+import { ApiKeyConfig } from '../api_key_configuration';
import { ConnectorNameAndDescription } from '../connector_name_and_description/connector_name_and_description';
import { BETA_CONNECTORS, NATIVE_CONNECTORS } from '../constants';
@@ -45,6 +47,7 @@ export const NativeConnectorConfiguration: React.FC = () => {
const { index } = useValues(IndexViewLogic);
const { config } = useValues(KibanaLogic);
const { errorConnectingMessage } = useValues(HttpLogic);
+ const { data: apiKeyData } = useValues(GenerateConnectorApiKeyApiLogic);
if (!isConnectorIndex(index)) {
return <>;
@@ -74,6 +77,8 @@ export const NativeConnectorConfiguration: React.FC = () => {
const hasResearched = hasDescription || hasConfigured || hasConfiguredAdvanced;
const icon = nativeConnector.icon;
+ const hasApiKey = !!(index.connector.api_key_id ?? apiKeyData);
+
// TODO service_type === "" is considered unknown/custom connector multipleplaces replace all of them with a better solution
const isBeta =
!index.connector.service_type ||
@@ -140,6 +145,24 @@ export const NativeConnectorConfiguration: React.FC = () => {
),
titleSize: 'xs',
},
+ {
+ children: (
+
+ ),
+ status: hasApiKey ? 'complete' : 'incomplete',
+ title: i18n.translate(
+ 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.regenerateApiKeyTitle',
+ {
+ defaultMessage: 'Regenerate API key',
+ }
+ ),
+ titleSize: 'xs',
+ },
{
children: ,
status: hasDescription ? 'complete' : 'incomplete',
diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts
index de2d2d2db3927..61c250ebc27e5 100644
--- a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts
+++ b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.test.ts
@@ -11,8 +11,6 @@ import {
createConnector,
fetchConnectorByIndexName,
deleteConnectorById,
- createConnectorSecret,
- updateConnectorApiKeyId,
} from '@kbn/search-connectors';
import { ErrorCode } from '../../../common/types/error_codes';
@@ -27,8 +25,6 @@ jest.mock('@kbn/search-connectors', () => ({
createConnector: jest.fn(),
deleteConnectorById: jest.fn(),
fetchConnectorByIndexName: jest.fn(),
- createConnectorSecret: jest.fn(),
- updateConnectorApiKeyId: jest.fn(),
}));
jest.mock('../crawler/fetch_crawlers', () => ({ fetchCrawlerByIndexName: jest.fn() }));
jest.mock('../indices/generate_api_key', () => ({ generateApiKey: jest.fn() }));
@@ -76,10 +72,7 @@ describe('addConnector lib function', () => {
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => undefined);
(fetchCrawlerByIndexName as jest.Mock).mockImplementation(() => undefined);
mockClient.asCurrentUser.indices.getMapping.mockImplementation(() => connectorsIndicesMapping);
-
(generateApiKey as jest.Mock).mockImplementation(() => undefined);
- (createConnectorSecret as jest.Mock).mockImplementation(() => undefined);
- (updateConnectorApiKeyId as jest.Mock).mockImplementation(() => undefined);
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, {
@@ -108,8 +101,6 @@ describe('addConnector lib function', () => {
// non-native connector should not generate API key or update secrets storage
expect(generateApiKey).toBeCalledTimes(0);
- expect(createConnectorSecret).toBeCalledTimes(0);
- expect(updateConnectorApiKeyId).toBeCalledTimes(0);
});
it('should add a native connector', async () => {
@@ -122,13 +113,10 @@ describe('addConnector lib function', () => {
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => undefined);
(fetchCrawlerByIndexName as jest.Mock).mockImplementation(() => undefined);
mockClient.asCurrentUser.indices.getMapping.mockImplementation(() => connectorsIndicesMapping);
-
(generateApiKey as jest.Mock).mockImplementation(() => ({
id: 'api-key-id',
encoded: 'encoded-api-key',
}));
- (createConnectorSecret as jest.Mock).mockImplementation(() => ({ id: 'connector-secret-id' }));
- (updateConnectorApiKeyId as jest.Mock).mockImplementation(() => ({ acknowledged: true }));
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, {
@@ -156,14 +144,7 @@ describe('addConnector lib function', () => {
});
// native connector should generate API key and update secrets storage
- expect(generateApiKey).toHaveBeenCalledWith(mockClient, 'index_name');
- expect(createConnectorSecret).toHaveBeenCalledWith(mockClient.asCurrentUser, 'encoded-api-key');
- expect(updateConnectorApiKeyId).toHaveBeenCalledWith(
- mockClient.asCurrentUser,
- 'fakeId',
- 'api-key-id',
- 'connector-secret-id'
- );
+ expect(generateApiKey).toHaveBeenCalledWith(mockClient, 'index_name', true, null);
});
it('should reject if index already exists', async () => {
@@ -254,13 +235,10 @@ describe('addConnector lib function', () => {
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => ({ id: 'connectorId' }));
(fetchCrawlerByIndexName as jest.Mock).mockImplementation(() => undefined);
mockClient.asCurrentUser.indices.getMapping.mockImplementation(() => connectorsIndicesMapping);
-
(generateApiKey as jest.Mock).mockImplementation(() => ({
id: 'api-key-id',
encoded: 'encoded-api-key',
}));
- (createConnectorSecret as jest.Mock).mockImplementation(() => ({ id: 'connector-secret-id' }));
- (updateConnectorApiKeyId as jest.Mock).mockImplementation(() => ({ acknowledged: true }));
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, {
diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts
index 3c5265234bb9a..0f15f2767079f 100644
--- a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts
+++ b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts
@@ -9,11 +9,9 @@ import { IScopedClusterClient } from '@kbn/core/server';
import {
createConnector,
- createConnectorSecret,
Connector,
ConnectorStatus,
deleteConnectorById,
- updateConnectorApiKeyId,
} from '@kbn/search-connectors';
import { fetchConnectorByIndexName, NATIVE_CONNECTOR_DEFINITIONS } from '@kbn/search-connectors';
@@ -97,14 +95,7 @@ export const addConnector = async (
input.isNative &&
input.serviceType !== ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE
) {
- const apiKey = await generateApiKey(client, index);
- const connectorSecret = await createConnectorSecret(client.asCurrentUser, apiKey.encoded);
- await updateConnectorApiKeyId(
- client.asCurrentUser,
- connector.id,
- apiKey.id,
- connectorSecret.id
- );
+ await generateApiKey(client, index, true, null);
}
return connector;
diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts
index 92c87b354a470..541566ee2d19d 100644
--- a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts
+++ b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts
@@ -7,11 +7,22 @@
import { IScopedClusterClient } from '@kbn/core/server';
-import { CONNECTORS_INDEX } from '@kbn/search-connectors';
+import {
+ CONNECTORS_INDEX,
+ createConnectorSecret,
+ updateConnectorSecret,
+} from '@kbn/search-connectors';
import { generateApiKey } from './generate_api_key';
-describe('generateApiKey lib function', () => {
+jest.mock('@kbn/search-connectors', () => ({
+ CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX: '.search-acl-filter-',
+ CONNECTORS_INDEX: '.elastic-connectors',
+ createConnectorSecret: jest.fn(),
+ updateConnectorSecret: jest.fn(),
+}));
+
+describe('generateApiKey lib function for connector clients', () => {
const mockClient = {
asCurrentUser: {
index: jest.fn(),
@@ -47,9 +58,11 @@ describe('generateApiKey lib function', () => {
encoded: 'encoded',
id: 'apiKeyId',
}));
+ (createConnectorSecret as jest.Mock).mockImplementation(() => undefined);
+ (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined);
await expect(
- generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name')
+ generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', false, null)
).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' });
expect(mockClient.asCurrentUser.index).not.toHaveBeenCalled();
expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({
@@ -66,6 +79,8 @@ describe('generateApiKey lib function', () => {
},
},
});
+ expect(createConnectorSecret).toBeCalledTimes(0);
+ expect(updateConnectorSecret).toBeCalledTimes(0);
});
it('should create an API key plus connector for connectors', async () => {
mockClient.asCurrentUser.search.mockImplementation(() =>
@@ -83,9 +98,11 @@ describe('generateApiKey lib function', () => {
encoded: 'encoded',
id: 'apiKeyId',
}));
+ (createConnectorSecret as jest.Mock).mockImplementation(() => undefined);
+ (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined);
await expect(
- generateApiKey(mockClient as unknown as IScopedClusterClient, 'search-test')
+ generateApiKey(mockClient as unknown as IScopedClusterClient, 'search-test', false, null)
).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' });
expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({
name: 'search-test-connector',
@@ -107,6 +124,8 @@ describe('generateApiKey lib function', () => {
index: CONNECTORS_INDEX,
});
expect(mockClient.asCurrentUser.security.invalidateApiKey).not.toHaveBeenCalled();
+ expect(createConnectorSecret).toBeCalledTimes(0);
+ expect(updateConnectorSecret).toBeCalledTimes(0);
});
it('should invalidate API key if already defined', async () => {
mockClient.asCurrentUser.search.mockImplementation(() =>
@@ -130,9 +149,11 @@ describe('generateApiKey lib function', () => {
encoded: 'encoded',
id: 'apiKeyId',
}));
+ (createConnectorSecret as jest.Mock).mockImplementation(() => undefined);
+ (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined);
await expect(
- generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name')
+ generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', false, null)
).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' });
expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({
name: 'index_name-connector',
@@ -154,7 +175,173 @@ describe('generateApiKey lib function', () => {
index: CONNECTORS_INDEX,
});
expect(mockClient.asCurrentUser.security.invalidateApiKey).toHaveBeenCalledWith({
- id: '1',
+ ids: ['1'],
+ });
+ expect(createConnectorSecret).toBeCalledTimes(0);
+ expect(updateConnectorSecret).toBeCalledTimes(0);
+ });
+});
+
+describe('generateApiKey lib function for native connectors', () => {
+ const mockClient = {
+ asCurrentUser: {
+ index: jest.fn(),
+ indices: {
+ create: jest.fn(),
+ },
+ search: jest.fn(),
+ security: {
+ createApiKey: jest.fn(),
+ invalidateApiKey: jest.fn(),
+ },
+ },
+ asInternalUser: {},
+ };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it('should create an API key if index does not have a connector', async () => {
+ mockClient.asCurrentUser.search.mockImplementation(() =>
+ Promise.resolve({
+ hits: {
+ hits: [],
+ },
+ })
+ );
+ mockClient.asCurrentUser.index.mockImplementation(() => ({
+ _id: 'connectorId',
+ _source: 'Document',
+ }));
+ mockClient.asCurrentUser.security.createApiKey.mockImplementation(() => ({
+ encoded: 'encoded',
+ id: 'apiKeyId',
+ }));
+ (createConnectorSecret as jest.Mock).mockImplementation(() => undefined);
+ (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined);
+
+ await expect(
+ generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', true, null)
+ ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' });
+ expect(mockClient.asCurrentUser.index).not.toHaveBeenCalled();
+ expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({
+ name: 'index_name-connector',
+ role_descriptors: {
+ ['index-name-connector-role']: {
+ cluster: ['monitor'],
+ index: [
+ {
+ names: ['index_name', '.search-acl-filter-index_name', `${CONNECTORS_INDEX}*`],
+ privileges: ['all'],
+ },
+ ],
+ },
+ },
+ });
+ expect(createConnectorSecret).toBeCalledTimes(0);
+ expect(updateConnectorSecret).toBeCalledTimes(0);
+ });
+ it('should create an API key plus connector for connectors', async () => {
+ mockClient.asCurrentUser.search.mockImplementation(() =>
+ Promise.resolve({
+ hits: {
+ hits: [{ _id: 'connectorId', _source: { doc: 'doc' } }],
+ },
+ })
+ );
+ mockClient.asCurrentUser.index.mockImplementation(() => ({
+ _id: 'connectorId',
+ _source: 'Document',
+ }));
+ mockClient.asCurrentUser.security.createApiKey.mockImplementation(() => ({
+ encoded: 'encoded',
+ id: 'apiKeyId',
+ }));
+ (createConnectorSecret as jest.Mock).mockImplementation(() => ({
+ id: '1234',
+ }));
+ (updateConnectorSecret as jest.Mock).mockImplementation(() => undefined);
+
+ await expect(
+ generateApiKey(mockClient as unknown as IScopedClusterClient, 'search-test', true, null)
+ ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' });
+ expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({
+ name: 'search-test-connector',
+ role_descriptors: {
+ ['search-test-connector-role']: {
+ cluster: ['monitor'],
+ index: [
+ {
+ names: ['search-test', '.search-acl-filter-search-test', `${CONNECTORS_INDEX}*`],
+ privileges: ['all'],
+ },
+ ],
+ },
+ },
+ });
+ expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({
+ document: { api_key_id: 'apiKeyId', api_key_secret_id: '1234', doc: 'doc' },
+ id: 'connectorId',
+ index: CONNECTORS_INDEX,
+ });
+ expect(mockClient.asCurrentUser.security.invalidateApiKey).not.toHaveBeenCalled();
+ expect(createConnectorSecret).toHaveBeenCalledWith(mockClient.asCurrentUser, 'encoded');
+ expect(updateConnectorSecret).toBeCalledTimes(0);
+ });
+ it('should invalidate API key if already defined', async () => {
+ mockClient.asCurrentUser.search.mockImplementation(() =>
+ Promise.resolve({
+ hits: {
+ hits: [
+ {
+ _id: 'connectorId',
+ _source: { api_key_id: '1', doc: 'doc' },
+ fields: { api_key_id: '1' },
+ },
+ ],
+ },
+ })
+ );
+ mockClient.asCurrentUser.index.mockImplementation(() => ({
+ _id: 'connectorId',
+ _source: 'Document',
+ }));
+ mockClient.asCurrentUser.security.createApiKey.mockImplementation(() => ({
+ encoded: 'encoded',
+ id: 'apiKeyId',
+ }));
+ (createConnectorSecret as jest.Mock).mockImplementation(() => undefined);
+ (updateConnectorSecret as jest.Mock).mockImplementation(() => ({
+ result: 'updated',
+ }));
+
+ await expect(
+ generateApiKey(mockClient as unknown as IScopedClusterClient, 'index_name', true, '1234')
+ ).resolves.toEqual({ encoded: 'encoded', id: 'apiKeyId' });
+ expect(mockClient.asCurrentUser.security.createApiKey).toHaveBeenCalledWith({
+ name: 'index_name-connector',
+ role_descriptors: {
+ ['index-name-connector-role']: {
+ cluster: ['monitor'],
+ index: [
+ {
+ names: ['index_name', '.search-acl-filter-index_name', `${CONNECTORS_INDEX}*`],
+ privileges: ['all'],
+ },
+ ],
+ },
+ },
+ });
+ expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({
+ document: { api_key_id: 'apiKeyId', api_key_secret_id: '1234', doc: 'doc' },
+ id: 'connectorId',
+ index: CONNECTORS_INDEX,
+ });
+ expect(mockClient.asCurrentUser.security.invalidateApiKey).toHaveBeenCalledWith({
+ ids: ['1'],
});
+ expect(createConnectorSecret).toBeCalledTimes(0);
+ expect(updateConnectorSecret).toHaveBeenCalledWith(mockClient.asCurrentUser, 'encoded', '1234');
});
});
diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts
index fb2ddbaad9f9d..01955cb004b24 100644
--- a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts
+++ b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts
@@ -11,11 +11,18 @@ import {
ConnectorDocument,
CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX,
CONNECTORS_INDEX,
+ createConnectorSecret,
+ updateConnectorSecret,
} from '@kbn/search-connectors';
import { toAlphanumeric } from '../../../common/utils/to_alphanumeric';
-export const generateApiKey = async (client: IScopedClusterClient, indexName: string) => {
+export const generateApiKey = async (
+ client: IScopedClusterClient,
+ indexName: string,
+ isNative: boolean,
+ secretId: string | null
+) => {
const aclIndexName = `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}${indexName}`;
const apiKeyResult = await client.asCurrentUser.security.createApiKey({
@@ -32,20 +39,47 @@ export const generateApiKey = async (client: IScopedClusterClient, indexName: st
},
},
});
+
const connectorResult = await client.asCurrentUser.search({
index: CONNECTORS_INDEX,
query: { term: { index_name: indexName } },
});
const connector = connectorResult.hits.hits[0];
if (connector) {
- if (connector.fields?.api_key_id) {
- await client.asCurrentUser.security.invalidateApiKey({ id: connector.fields.api_key_id });
+ const apiKeyFields = isNative
+ ? {
+ api_key_id: apiKeyResult.id,
+ api_key_secret_id: await storeConnectorSecret(client, apiKeyResult.encoded, secretId),
+ }
+ : {
+ api_key_id: apiKeyResult.id,
+ };
+
+ if (connector._source?.api_key_id) {
+ await client.asCurrentUser.security.invalidateApiKey({ ids: [connector._source.api_key_id] });
}
await client.asCurrentUser.index({
- document: { ...connector._source, api_key_id: apiKeyResult.id },
+ document: {
+ ...connector._source,
+ ...apiKeyFields,
+ },
id: connector._id,
index: CONNECTORS_INDEX,
});
}
return apiKeyResult;
};
+
+const storeConnectorSecret = async (
+ client: IScopedClusterClient,
+ value: string,
+ secretId: string | null
+) => {
+ if (secretId === null) {
+ const connectorSecretResult = await createConnectorSecret(client.asCurrentUser, value);
+ return connectorSecretResult.id;
+ }
+
+ await updateConnectorSecret(client.asCurrentUser, value, secretId);
+ return secretId;
+};
diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts
index fdbeca1e82ff9..9872f4d7c7a66 100644
--- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts
+++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts
@@ -278,6 +278,10 @@ export function registerIndexRoutes({
{
path: '/internal/enterprise_search/indices/{indexName}/api_key',
validate: {
+ body: schema.object({
+ is_native: schema.boolean(),
+ secret_id: schema.maybe(schema.nullable(schema.string())),
+ }),
params: schema.object({
indexName: schema.string(),
}),
@@ -285,9 +289,11 @@ export function registerIndexRoutes({
},
elasticsearchErrorHandler(log, async (context, request, response) => {
const indexName = decodeURIComponent(request.params.indexName);
+ const { is_native: isNative, secret_id: secretId } = request.body;
+
const { client } = (await context.core).elasticsearch;
- const apiKey = await generateApiKey(client, indexName);
+ const apiKey = await generateApiKey(client, indexName, isNative, secretId || null);
return response.ok({
body: apiKey,
From 2c0fd46961677c913928a452ef5073feeec8f220 Mon Sep 17 00:00:00 2001
From: Elena Stoeva <59341489+ElenaStoeva@users.noreply.github.com>
Date: Fri, 9 Feb 2024 11:07:18 +0000
Subject: [PATCH 13/13] [Index Management] Fix index template simulate request
(#176183)
Fixes https://github.com/elastic/kibana/issues/165889
## Summary
This PR fixes the simulate request for index templates. Previously, it
would fail when we simulate an index template which references a
component template that is not created yet, because the
`ignore_missing_component_templates` option wasn't configured. This PR
adds this property to the template format so that it is sent as part of
the simulate request.
**How to test:**
1. Start Es with `yarn es snapshot` and Kibana with `yarn start`.
2. Go to Dev Tools.
3. Create an index template that references a component template:
```
PUT _index_template/test-template
{
"index_patterns": ["test-*"],
"composed_of": ["some_component_template"],
"ignore_missing_component_templates": ["some_component_template"]
}
```
4. Go to Stack Management -> Index Management -> Index Templates.
5. Click on the created index template to open the details flyout.
6. Click on the "Preview" tab and verify that the simulate request was
successful.
---
.../index_management/common/lib/template_serialization.ts | 4 ++++
x-pack/plugins/index_management/common/types/templates.ts | 2 ++
.../server/routes/api/templates/validate_schemas.ts | 1 +
.../apis/management/index_management/templates.ts | 6 ++++++
.../test_suites/common/index_management/index_templates.ts | 2 ++
5 files changed, 15 insertions(+)
diff --git a/x-pack/plugins/index_management/common/lib/template_serialization.ts b/x-pack/plugins/index_management/common/lib/template_serialization.ts
index 5ec150a85aa17..80a5514969f54 100644
--- a/x-pack/plugins/index_management/common/lib/template_serialization.ts
+++ b/x-pack/plugins/index_management/common/lib/template_serialization.ts
@@ -23,6 +23,7 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T
indexPatterns,
template,
composedOf,
+ ignoreMissingComponentTemplates,
dataStream,
_meta,
allowAutoCreate,
@@ -35,6 +36,7 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T
index_patterns: indexPatterns,
data_stream: dataStream,
composed_of: composedOf,
+ ignore_missing_component_templates: ignoreMissingComponentTemplates,
allow_auto_create: allowAutoCreate,
_meta,
};
@@ -52,6 +54,7 @@ export function deserializeTemplate(
priority,
_meta,
composed_of: composedOf,
+ ignore_missing_component_templates: ignoreMissingComponentTemplates,
data_stream: dataStream,
deprecated,
allow_auto_create: allowAutoCreate,
@@ -76,6 +79,7 @@ export function deserializeTemplate(
template,
ilmPolicy: settings?.index?.lifecycle,
composedOf: composedOf ?? [],
+ ignoreMissingComponentTemplates: ignoreMissingComponentTemplates ?? [],
dataStream,
allowAutoCreate,
_meta,
diff --git a/x-pack/plugins/index_management/common/types/templates.ts b/x-pack/plugins/index_management/common/types/templates.ts
index e2f530b4ad502..9205605c70010 100644
--- a/x-pack/plugins/index_management/common/types/templates.ts
+++ b/x-pack/plugins/index_management/common/types/templates.ts
@@ -23,6 +23,7 @@ export interface TemplateSerialized {
};
deprecated?: boolean;
composed_of?: string[];
+ ignore_missing_component_templates?: string[];
version?: number;
priority?: number;
_meta?: { [key: string]: any };
@@ -45,6 +46,7 @@ export interface TemplateDeserialized {
};
lifecycle?: DataRetention;
composedOf?: string[]; // Composable template only
+ ignoreMissingComponentTemplates?: string[];
version?: number;
priority?: number; // Composable template only
allowAutoCreate?: boolean;
diff --git a/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts b/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts
index b9020585ed676..1f5c5e2a3b82e 100644
--- a/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts
+++ b/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts
@@ -28,6 +28,7 @@ export const templateSchema = schema.object({
})
),
composedOf: schema.maybe(schema.arrayOf(schema.string())),
+ ignoreMissingComponentTemplates: schema.maybe(schema.arrayOf(schema.string())),
dataStream: schema.maybe(
schema.object(
{
diff --git a/x-pack/test/api_integration/apis/management/index_management/templates.ts b/x-pack/test/api_integration/apis/management/index_management/templates.ts
index a3b2ce50e04b5..6d0e79993d040 100644
--- a/x-pack/test/api_integration/apis/management/index_management/templates.ts
+++ b/x-pack/test/api_integration/apis/management/index_management/templates.ts
@@ -96,6 +96,7 @@ export default function ({ getService }: FtrProviderContext) {
'hasMappings',
'priority',
'composedOf',
+ 'ignoreMissingComponentTemplates',
'version',
'_kbnMeta',
].sort();
@@ -119,6 +120,7 @@ export default function ({ getService }: FtrProviderContext) {
'version',
'_kbnMeta',
'composedOf',
+ 'ignoreMissingComponentTemplates',
].sort();
expect(Object.keys(legacyTemplateFound).sort()).to.eql(expectedLegacyKeys);
@@ -139,6 +141,7 @@ export default function ({ getService }: FtrProviderContext) {
'hasMappings',
'priority',
'composedOf',
+ 'ignoreMissingComponentTemplates',
'dataStream',
'version',
'_kbnMeta',
@@ -162,6 +165,7 @@ export default function ({ getService }: FtrProviderContext) {
'hasMappings',
'priority',
'composedOf',
+ 'ignoreMissingComponentTemplates',
'version',
'_kbnMeta',
].sort();
@@ -183,6 +187,7 @@ export default function ({ getService }: FtrProviderContext) {
'indexPatterns',
'template',
'composedOf',
+ 'ignoreMissingComponentTemplates',
'priority',
'version',
'_kbnMeta',
@@ -207,6 +212,7 @@ export default function ({ getService }: FtrProviderContext) {
'version',
'_kbnMeta',
'composedOf',
+ 'ignoreMissingComponentTemplates',
].sort();
const expectedTemplateKeys = ['aliases', 'mappings', 'settings'].sort();
diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts
index 6546d94afe391..82fd6dd057ae5 100644
--- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts
+++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts
@@ -104,6 +104,7 @@ export default function ({ getService }: FtrProviderContext) {
'hasMappings',
'_kbnMeta',
'composedOf',
+ 'ignoreMissingComponentTemplates',
].sort();
expect(Object.keys(indexTemplateFound).sort()).to.eql(expectedKeys);
@@ -124,6 +125,7 @@ export default function ({ getService }: FtrProviderContext) {
'template',
'_kbnMeta',
'composedOf',
+ 'ignoreMissingComponentTemplates',
].sort();
expect(body.name).to.eql(templateName);