diff --git a/docs/developer/contributing/development-functional-tests.asciidoc b/docs/developer/contributing/development-functional-tests.asciidoc index 2d47ad92582bb..cf854c6d53cfa 100644 --- a/docs/developer/contributing/development-functional-tests.asciidoc +++ b/docs/developer/contributing/development-functional-tests.asciidoc @@ -31,18 +31,23 @@ There are three ways to run the tests depending on your goals: 3. Custom option: ** Description: Runs tests against instances of {es} & {kib} started some other way (like Elastic Cloud, or an instance you are managing in some other way). -** just executes the functional tests -** url, credentials, etc. for {es} and {kib} are specified via environment variables -** Here's an example that runs against an Elastic Cloud instance. Note that you must run the same branch of tests as the version of {kib} you're testing. +** Just executes the functional tests +** URL, credentials, etc. for {es} and {kib} are specified via environment variables +** When running against an Elastic Cloud instance, additional environment variables are required `TEST_CLOUD` and `ES_SECURITY_ENABLED` +** You must run the same branch of tests as the version of {kib} you're testing. To run against a previous minor version use option `--es-version ` +** To run a specific configuration use option `--config ` +** Here's an example that runs against an Elastic Cloud instance + ["source","shell"] ---------- -export TEST_KIBANA_URL=https://kibana:password@my-kibana-instance.internal.net:443 +export TEST_KIBANA_URL=https://elastic:password@my-kbn-cluster.elastic-cloud.com:443 +export TEST_ES_URL=https://elastic:password@my-es-cluster.elastic-cloud.com:443 -export TEST_ES_URL=https://elastic:password@my-es-cluster.internal.net:9200 -node scripts/functional_test_runner ----------- +export TEST_CLOUD=1 +export ES_SECURITY_ENABLED=1 +node scripts/functional_test_runner [--config ] [--es-version ] +---------- ** Or you can override any or all of these individual parts of the URL and leave the others to the default values. + @@ -527,4 +532,4 @@ If your functional tests are flaky then the Operations team might skip them and This will take you to Buildkite where your build will run and tell you if it failed in any execution. -A flaky test may only fail once in 1000 runs, so keep this in mind and make sure you use enough executions to really prove that a test isn't flaky anymore. \ No newline at end of file +A flaky test may only fail once in 1000 runs, so keep this in mind and make sure you use enough executions to really prove that a test isn't flaky anymore. diff --git a/x-pack/packages/ml/agg_utils/index.ts b/x-pack/packages/ml/agg_utils/index.ts index ac51405d0b8eb..cc7a426f94050 100644 --- a/x-pack/packages/ml/agg_utils/index.ts +++ b/x-pack/packages/ml/agg_utils/index.ts @@ -16,6 +16,7 @@ export type { AggCardinality, ChangePoint, ChangePointGroup, + ChangePointGroupHistogram, ChangePointHistogram, ChangePointHistogramItem, HistogramField, diff --git a/x-pack/packages/ml/agg_utils/src/types.ts b/x-pack/packages/ml/agg_utils/src/types.ts index de993e4b9f321..a2c0d9f9a1569 100644 --- a/x-pack/packages/ml/agg_utils/src/types.ts +++ b/x-pack/packages/ml/agg_utils/src/types.ts @@ -87,6 +87,14 @@ export interface ChangePointHistogram extends FieldValuePair { histogram: ChangePointHistogramItem[]; } +/** + * Change point histogram data for a group of field/value pairs. + */ +export interface ChangePointGroupHistogram { + id: string; + histogram: ChangePointHistogramItem[]; +} + interface ChangePointGroupItem extends FieldValuePair { duplicate?: boolean; } @@ -95,7 +103,9 @@ interface ChangePointGroupItem extends FieldValuePair { * Tree leaves */ export interface ChangePointGroup { + id: string; group: ChangePointGroupItem[]; docCount: number; pValue: number | null; + histogram?: ChangePointHistogramItem[]; } diff --git a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts index 938c765d8e0d2..e050946a489be 100644 --- a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts +++ b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts @@ -5,12 +5,18 @@ * 2.0. */ -import type { ChangePoint, ChangePointHistogram, ChangePointGroup } from '@kbn/ml-agg-utils'; +import type { + ChangePoint, + ChangePointHistogram, + ChangePointGroup, + ChangePointGroupHistogram, +} from '@kbn/ml-agg-utils'; export const API_ACTION_NAME = { ADD_CHANGE_POINTS: 'add_change_points', ADD_CHANGE_POINTS_HISTOGRAM: 'add_change_points_histogram', ADD_CHANGE_POINTS_GROUP: 'add_change_point_group', + ADD_CHANGE_POINTS_GROUP_HISTOGRAM: 'add_change_point_group_histogram', ADD_ERROR: 'add_error', RESET: 'reset', UPDATE_LOADING_STATE: 'update_loading_state', @@ -57,6 +63,20 @@ export function addChangePointsGroupAction(payload: ApiActionAddChangePointsGrou }; } +interface ApiActionAddChangePointsGroupHistogram { + type: typeof API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM; + payload: ChangePointGroupHistogram[]; +} + +export function addChangePointsGroupHistogramAction( + payload: ApiActionAddChangePointsGroupHistogram['payload'] +): ApiActionAddChangePointsGroupHistogram { + return { + type: API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM, + payload, + }; +} + interface ApiActionAddError { type: typeof API_ACTION_NAME.ADD_ERROR; payload: string; @@ -99,6 +119,7 @@ export type AiopsExplainLogRateSpikesApiAction = | ApiActionAddChangePoints | ApiActionAddChangePointsGroup | ApiActionAddChangePointsHistogram + | ApiActionAddChangePointsGroupHistogram | ApiActionAddError | ApiActionReset | ApiActionUpdateLoadingState; diff --git a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts index dbc6be43766c6..5628b509980ad 100644 --- a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts +++ b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts @@ -8,6 +8,7 @@ export { addChangePointsAction, addChangePointsGroupAction, + addChangePointsGroupHistogramAction, addChangePointsHistogramAction, addErrorAction, resetAction, diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.ts b/x-pack/plugins/aiops/common/api/stream_reducer.ts index ff275d1414e91..690db961f5121 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.ts @@ -51,6 +51,15 @@ export function streamReducer( return { ...state, changePoints }; case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP: return { ...state, changePointsGroups: action.payload }; + case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM: + const changePointsGroups = state.changePointsGroups.map((cpg) => { + const cpHistogram = action.payload.find((h) => h.id === cpg.id); + if (cpHistogram) { + cpg.histogram = cpHistogram.histogram; + } + return cpg; + }); + return { ...state, changePointsGroups }; case API_ACTION_NAME.ADD_ERROR: return { ...state, errors: [...state.errors, action.payload] }; case API_ACTION_NAME.RESET: diff --git a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx index 350ab9f2e0205..b317ac6f2fed8 100644 --- a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx +++ b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx @@ -127,7 +127,7 @@ export const ExplainLogRateSpikesAnalysis: FC }, []); const groupTableItems = useMemo(() => { - const tableItems = data.changePointsGroups.map(({ group, docCount, pValue }, index) => { + const tableItems = data.changePointsGroups.map(({ id, group, docCount, histogram, pValue }) => { const sortedGroup = group.sort((a, b) => a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0 ); @@ -144,11 +144,12 @@ export const ExplainLogRateSpikesAnalysis: FC }); return { - id: index, + id, docCount, pValue, group: dedupedGroup, repeatedValues, + histogram, }; }); diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx index a046250db20b2..37563fd2d43a0 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx @@ -24,7 +24,11 @@ import { import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { ChangePoint } from '@kbn/ml-agg-utils'; + import { useEuiTheme } from '../../hooks/use_eui_theme'; + +import { MiniHistogram } from '../mini_histogram'; + import { SpikeAnalysisTable } from './spike_analysis_table'; const NARROW_COLUMN_WIDTH = '120px'; @@ -36,11 +40,12 @@ const DEFAULT_SORT_FIELD = 'pValue'; const DEFAULT_SORT_DIRECTION = 'asc'; interface GroupTableItem { - id: number; + id: string; docCount: number; pValue: number | null; group: Record; repeatedValues: Record; + histogram: ChangePoint['histogram']; } interface SpikeAnalysisTableProps { @@ -196,6 +201,39 @@ export const SpikeAnalysisGroupsTable: FC = ({ sortable: false, textOnly: true, }, + { + 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnLogRate', + width: NARROW_COLUMN_WIDTH, + field: 'pValue', + name: ( + + <> + + + + + ), + render: (_, { histogram, id }) => ( + + ), + sortable: false, + }, { 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnPValue', width: NARROW_COLUMN_WIDTH, @@ -226,9 +264,12 @@ export const SpikeAnalysisGroupsTable: FC = ({ { 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnDocCount', field: 'docCount', - name: i18n.translate('xpack.aiops.correlations.spikeAnalysisTableGroups.docCountLabel', { - defaultMessage: 'Doc count', - }), + name: i18n.translate( + 'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.docCountLabel', + { + defaultMessage: 'Doc count', + } + ), sortable: true, width: '20%', }, @@ -281,6 +322,7 @@ export const SpikeAnalysisGroupsTable: FC = ({ compressed columns={columns} items={pageOfItems} + itemId="id" itemIdToExpandedRowMap={itemIdToExpandedRowMap} onChange={onChange} pagination={pagination} diff --git a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts index a6fecc0e1a870..48ea2dbddb1c3 100644 --- a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts +++ b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts @@ -18,10 +18,12 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import { streamFactory } from '@kbn/aiops-utils'; import type { ChangePoint, NumericChartData, NumericHistogramField } from '@kbn/ml-agg-utils'; import { fetchHistogramsForFields } from '@kbn/ml-agg-utils'; +import { stringHash } from '@kbn/ml-string-hash'; import { addChangePointsAction, addChangePointsGroupAction, + addChangePointsGroupHistogramAction, addChangePointsHistogramAction, aiopsExplainLogRateSpikesSchema, addErrorAction, @@ -216,6 +218,21 @@ export const defineExplainLogRateSpikesRoute = ( return; } + const histogramFields: [NumericHistogramField] = [ + { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, + ]; + + const [overallTimeSeries] = (await fetchHistogramsForFields( + client, + request.body.index, + { match_all: {} }, + // fields + histogramFields, + // samplerShardSize + -1, + undefined + )) as [NumericChartData]; + if (groupingEnabled) { // To optimize the `frequent_items` query, we identify duplicate change points by count attributes. // Note this is a compromise and not 100% accurate because there could be change points that @@ -325,27 +342,40 @@ export const defineExplainLogRateSpikesRoute = ( }); changePointGroups.push( - ...missingChangePoints.map((cp) => { + ...missingChangePoints.map(({ fieldName, fieldValue, doc_count: docCount, pValue }) => { const duplicates = groupedChangePoints.find((d) => - d.group.some( - (dg) => dg.fieldName === cp.fieldName && dg.fieldValue === cp.fieldValue - ) + d.group.some((dg) => dg.fieldName === fieldName && dg.fieldValue === fieldValue) ); if (duplicates !== undefined) { return { + id: `${stringHash( + JSON.stringify( + duplicates.group.map((d) => ({ + fieldName: d.fieldName, + fieldValue: d.fieldValue, + })) + ) + )}`, group: duplicates.group.map((d) => ({ fieldName: d.fieldName, fieldValue: d.fieldValue, duplicate: false, })), - docCount: cp.doc_count, - pValue: cp.pValue, + docCount, + pValue, }; } else { return { - group: [{ fieldName: cp.fieldName, fieldValue: cp.fieldValue, duplicate: false }], - docCount: cp.doc_count, - pValue: cp.pValue, + id: `${stringHash(JSON.stringify({ fieldName, fieldValue }))}`, + group: [ + { + fieldName, + fieldValue, + duplicate: false, + }, + ], + docCount, + pValue, }; } }) @@ -358,22 +388,62 @@ export const defineExplainLogRateSpikesRoute = ( if (maxItems > 1) { push(addChangePointsGroupAction(changePointGroups)); } - } - const histogramFields: [NumericHistogramField] = [ - { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, - ]; + if (changePointGroups) { + await asyncForEach(changePointGroups, async (cpg, index) => { + const histogramQuery = { + bool: { + filter: cpg.group.map((d) => ({ + term: { [d.fieldName]: d.fieldValue }, + })), + }, + }; - const [overallTimeSeries] = (await fetchHistogramsForFields( - client, - request.body.index, - { match_all: {} }, - // fields - histogramFields, - // samplerShardSize - -1, - undefined - )) as [NumericChartData]; + const [cpgTimeSeries] = (await fetchHistogramsForFields( + client, + request.body.index, + histogramQuery, + // fields + [ + { + fieldName: request.body.timeFieldName, + type: KBN_FIELD_TYPES.DATE, + interval: overallTimeSeries.interval, + min: overallTimeSeries.stats[0], + max: overallTimeSeries.stats[1], + }, + ], + // samplerShardSize + -1, + undefined + )) as [NumericChartData]; + + const histogram = + overallTimeSeries.data.map((o, i) => { + const current = cpgTimeSeries.data.find( + (d1) => d1.key_as_string === o.key_as_string + ) ?? { + doc_count: 0, + }; + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_change_point: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + }) ?? []; + + push( + addChangePointsGroupHistogramAction([ + { + id: cpg.id, + histogram, + }, + ]) + ); + }); + } + } // time series filtered by fields if (changePoints) { diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts index 1ff92181dee96..5f2125a583db7 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts @@ -11,6 +11,7 @@ import { getFieldValuePairCounts, markDuplicates } from './get_simple_hierarchic const changePointGroups: ChangePointGroup[] = [ { + id: 'group-1', group: [ { fieldName: 'custom_field.keyword', @@ -25,6 +26,7 @@ const changePointGroups: ChangePointGroup[] = [ pValue: 0.01, }, { + id: 'group-2', group: [ { fieldName: 'custom_field.keyword', @@ -64,6 +66,7 @@ describe('get_simple_hierarchical_tree', () => { expect(markedDuplicates).toEqual([ { + id: 'group-1', group: [ { fieldName: 'custom_field.keyword', @@ -80,6 +83,7 @@ describe('get_simple_hierarchical_tree', () => { pValue: 0.01, }, { + id: 'group-2', group: [ { fieldName: 'custom_field.keyword', diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts index bebbb1302376d..9f39d1eb11f68 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts @@ -8,6 +8,7 @@ // import { omit, uniq } from 'lodash'; import type { ChangePointGroup, FieldValuePair } from '@kbn/ml-agg-utils'; +import { stringHash } from '@kbn/ml-string-hash'; import type { ItemsetResult } from './fetch_frequent_items'; @@ -230,9 +231,13 @@ export function getSimpleHierarchicalTreeLeaves( leaves: ChangePointGroup[], level = 1 ) { - // console.log(`${'-'.repeat(level)} ${tree.name} ${tree.children.length}`); if (tree.children.length === 0) { - leaves.push({ group: tree.set, docCount: tree.docCount, pValue: tree.pValue }); + leaves.push({ + id: `${stringHash(JSON.stringify(tree.set))}`, + group: tree.set, + docCount: tree.docCount, + pValue: tree.pValue, + }); } else { for (const child of tree.children) { const newLeaves = getSimpleHierarchicalTreeLeaves(child, [], level + 1); diff --git a/x-pack/plugins/cloud_security_posture/public/components/csp_evaluation_badge.tsx b/x-pack/plugins/cloud_security_posture/public/components/csp_evaluation_badge.tsx index ab193ab6dc8be..0d838daa1e660 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/csp_evaluation_badge.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/csp_evaluation_badge.tsx @@ -13,6 +13,10 @@ interface Props { type: 'passed' | 'failed'; } +// 'fail' / 'pass' are same chars length, but not same width size. +// 46px is used to make sure the badge is always the same width. +const BADGE_WIDTH = 46; + const getColor = (type: Props['type']): EuiBadgeProps['color'] => { if (type === 'passed') return 'success'; if (type === 'failed') return 'danger'; @@ -20,7 +24,7 @@ const getColor = (type: Props['type']): EuiBadgeProps['color'] => { }; export const CspEvaluationBadge = ({ type }: Props) => ( - + {type === 'failed' ? ( ) : ( diff --git a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/findings_by_resource_table.tsx b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/findings_by_resource_table.tsx index 9be73dbe4809a..18ddb34256df9 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/findings_by_resource_table.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/findings_by_resource_table.tsx @@ -19,11 +19,14 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import numeral from '@elastic/numeral'; import { Link, generatePath } from 'react-router-dom'; -import { ColumnNameWithTooltip } from '../../../components/column_name_with_tooltip'; import * as TEST_SUBJECTS from '../test_subjects'; import type { FindingsByResourcePage } from './use_findings_by_resource'; import { findingsNavigation } from '../../../common/navigation/constants'; -import { createColumnWithFilters, type OnAddFilter } from '../layout/findings_layout'; +import { + createColumnWithFilters, + type OnAddFilter, + baseFindingsColumns, +} from '../layout/findings_layout'; export const formatNumber = (value: number) => value < 1000 ? value : numeral(value).format('0.0a'); @@ -101,55 +104,21 @@ const FindingsByResourceTableComponent = ({ const baseColumns: Array> = [ { + ...baseFindingsColumns['resource.id'], field: 'resource_id', - name: ( - - ), render: (resourceId: FindingsByResourcePage['resource_id']) => ( - + {resourceId} ), }, - { - field: 'resource.sub_type', - truncateText: true, - name: ( - - ), - }, - { - field: 'resource.name', - truncateText: true, - name: ( - - ), - }, - { - field: 'rule.benchmark.name', - truncateText: true, - name: ( - - ), - }, + baseFindingsColumns['resource.sub_type'], + baseFindingsColumns['resource.name'], + baseFindingsColumns['rule.benchmark.name'], { field: 'rule.section', truncateText: true, @@ -159,24 +128,16 @@ const baseColumns: Array> = defaultMessage="CIS Sections" /> ), - render: (sections: string[]) => sections.join(', '), - }, - { - field: 'cluster_id', - name: ( - - ), - truncateText: true, + render: (sections: string[]) => { + const items = sections.join(', '); + return ( + + <>{items} + + ); + }, }, + baseFindingsColumns.cluster_id, { field: 'failed_findings', width: '150px', diff --git a/x-pack/plugins/cloud_security_posture/public/pages/findings/layout/findings_layout.tsx b/x-pack/plugins/cloud_security_posture/public/pages/findings/layout/findings_layout.tsx index 8e158d47e615b..ff7be4d7429ad 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/findings/layout/findings_layout.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/findings/layout/findings_layout.tsx @@ -76,7 +76,7 @@ const baseColumns = [ /> ), truncateText: true, - width: '10%', + width: '150px', sortable: true, render: (filename: string) => ( @@ -89,7 +89,7 @@ const baseColumns = [ name: i18n.translate('xpack.csp.findings.findingsTable.findingsTableColumn.resultColumnLabel', { defaultMessage: 'Result', }), - width: '110px', + width: '120px', sortable: true, render: (type: PropsOf['type']) => ( @@ -103,6 +103,7 @@ const baseColumns = [ ), sortable: true, truncateText: true, + width: '10%', }, { field: 'resource.name', @@ -112,6 +113,11 @@ const baseColumns = [ ), sortable: true, truncateText: true, + render: (name: string) => ( + + <>{name} + + ), }, { field: 'rule.name', @@ -119,6 +125,11 @@ const baseColumns = [ defaultMessage: 'Rule', }), sortable: true, + render: (name: string) => ( + + <>{name} + + ), }, { field: 'rule.benchmark.name', @@ -126,6 +137,7 @@ const baseColumns = [ 'xpack.csp.findings.findingsTable.findingsTableColumn.ruleBenchmarkColumnLabel', { defaultMessage: 'Benchmark' } ), + width: '10%', sortable: true, truncateText: true, }, @@ -135,8 +147,14 @@ const baseColumns = [ 'xpack.csp.findings.findingsTable.findingsTableColumn.ruleSectionColumnLabel', { defaultMessage: 'CIS Section' } ), + width: '7%', sortable: true, truncateText: true, + render: (section: string) => ( + + <>{section} + + ), }, { field: 'rule.tags', @@ -144,7 +162,7 @@ const baseColumns = [ 'xpack.csp.findings.findingsTable.findingsTableColumn.ruleTagsColumnLabel', { defaultMessage: 'Rule Tags' } ), - width: '200px', + width: '15%', sortable: false, truncateText: true, render: (tags: string[]) => { @@ -166,13 +184,18 @@ const baseColumns = [ )} /> ), - width: '10%', - truncateText: true, + width: '150px', sortable: true, + truncateText: true, + render: (section: string) => ( + + <>{section} + + ), }, { field: '@timestamp', - width: '150px', + width: '10%', name: i18n.translate( 'xpack.csp.findings.findingsTable.findingsTableColumn.lastCheckedColumnLabel', { defaultMessage: 'Last Checked' } diff --git a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.test.tsx b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.test.tsx index f46454e45eb82..8d07e3aac6674 100644 --- a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.test.tsx @@ -9,9 +9,9 @@ import '../../../__mocks__/shallow_useeffect.mock'; import React from 'react'; -import { shallow } from 'enzyme'; +import { mount } from 'enzyme'; -import { EuiCodeBlock } from '@elastic/eui'; +import { EuiCodeBlock, EuiDescriptionListDescription } from '@elastic/eui'; import { AnalyticsCollection } from '../../../../../common/types/analytics'; @@ -29,7 +29,18 @@ describe('AnalyticsCollectionIntegrate', () => { }); it('renders', () => { - const wrapper = shallow(); + const wrapper = mount(); expect(wrapper.find(EuiCodeBlock)).toHaveLength(2); + expect(wrapper.find(EuiDescriptionListDescription).get(0)).toMatchInlineSnapshot(` + + example + + `); + + expect(wrapper.find(EuiDescriptionListDescription).get(1)).toMatchInlineSnapshot(` + + /analytics/api/collections/example + + `); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.tsx b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.tsx index ff5cdf9c5fee1..36b527097a304 100644 --- a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_integrate.tsx @@ -28,7 +28,7 @@ interface AnalyticsCollectionIntegrateProps { export const AnalyticsCollectionIntegrate: React.FC = ({ collection, }) => { - const analyticsDNSUrl = getEnterpriseSearchUrl(`/analytics/${collection.name}`); + const analyticsDNSUrl = getEnterpriseSearchUrl(`/analytics/api/collections/${collection.name}`); const credentials = [ { title: i18n.translate( @@ -111,9 +111,9 @@ export const AnalyticsCollectionIntegrate: React.FC - {`window.elasticAnalytics.trackEvent("ResultClick", { + {`window.elasticAnalytics.trackEvent("click", { title: "Website Analytics", - url: "www.elasitc.co/analytics/website" + url: "www.elastic.co/analytics/overview" })`} diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 60c4dd7fd8975..138279d7b7f5a 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -557,6 +557,18 @@ ] }, "savedObject": { + "type": "object" + }, + "latestVersion": { + "type": "string" + }, + "keepPoliciesUpToDate": { + "type": "boolean" + }, + "notice": { + "type": "string" + }, + "licensePath": { "type": "string" } }, @@ -4194,6 +4206,26 @@ "version": { "type": "string" }, + "release": { + "type": "string", + "enum": [ + "experimental", + "beta", + "ga" + ] + }, + "source": { + "type": "object", + "properties": { + "license": { + "type": "string", + "enum": [ + "Apache-2.0", + "Elastic-2.0" + ] + } + } + }, "readme": { "type": "string" }, @@ -4209,34 +4241,32 @@ "type": "string" } }, - "requirement": { - "oneOf": [ - { + "conditions": { + "type": "object", + "properties": { + "kibana": { + "type": "object", "properties": { - "kibana": { - "type": "object", - "properties": { - "versions": { - "type": "string" - } - } + "versions": { + "type": "string" } } }, - { + "elasticsearch": { + "type": "object", "properties": { - "elasticsearch": { - "type": "object", - "properties": { - "versions": { - "type": "string" - } - } + "subscription": { + "type": "string", + "enum": [ + "basic", + "gold", + "platinum", + "enterprise" + ] } } } - ], - "type": "object" + } }, "screenshots": { "type": "array", @@ -4340,6 +4370,22 @@ }, "path": { "type": "string" + }, + "elasticsearch": { + "type": "object", + "properties": { + "privileges": { + "type": "object", + "properties": { + "cluster": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } } }, "required": [ @@ -4349,7 +4395,7 @@ "description", "type", "categories", - "requirement", + "conditions", "assets", "format_version", "download", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 567ee06825b92..60c305097c4cc 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -350,6 +350,14 @@ paths: - install_failed - not_installed savedObject: + type: object + latestVersion: + type: string + keepPoliciesUpToDate: + type: boolean + notice: + type: string + licensePath: type: string required: - status @@ -2623,6 +2631,20 @@ components: type: string version: type: string + release: + type: string + enum: + - experimental + - beta + - ga + source: + type: object + properties: + license: + type: string + enum: + - Apache-2.0 + - Elastic-2.0 readme: type: string description: @@ -2633,21 +2655,24 @@ components: type: array items: type: string - requirement: - oneOf: - - properties: - kibana: - type: object - properties: - versions: - type: string - - properties: - elasticsearch: - type: object - properties: - versions: - type: string + conditions: type: object + properties: + kibana: + type: object + properties: + versions: + type: string + elasticsearch: + type: object + properties: + subscription: + type: string + enum: + - basic + - gold + - platinum + - enterprise screenshots: type: array items: @@ -2718,6 +2743,16 @@ components: type: string path: type: string + elasticsearch: + type: object + properties: + privileges: + type: object + properties: + cluster: + type: array + items: + type: string required: - name - title @@ -2725,7 +2760,7 @@ components: - description - type - categories - - requirement + - conditions - assets - format_version - download diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/package_info.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/package_info.yaml index e61c349f3f490..80c777a07718c 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/package_info.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/package_info.yaml @@ -7,6 +7,20 @@ properties: type: string version: type: string + release: + type: string + enum: + - experimental + - beta + - ga + source: + type: object + properties: + license: + type: string + enum: + - Apache-2.0 + - Elastic-2.0 readme: type: string description: @@ -17,21 +31,24 @@ properties: type: array items: type: string - requirement: - oneOf: - - properties: - kibana: - type: object - properties: - versions: - type: string - - properties: - elasticsearch: - type: object - properties: - versions: - type: string + conditions: type: object + properties: + kibana: + type: object + properties: + versions: + type: string + elasticsearch: + type: object + properties: + subscription: + type: string + enum: + - basic + - gold + - platinum + - enterprise screenshots: type: array items: @@ -102,6 +119,16 @@ properties: type: string path: type: string + elasticsearch: + type: object + properties: + privileges: + type: object + properties: + cluster: + type: array + items: + type: string required: - name - title @@ -109,7 +136,7 @@ required: - description - type - categories - - requirement + - conditions - assets - format_version - download diff --git a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml index 6ef61788acd62..8164d04fc98f1 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml @@ -21,6 +21,14 @@ get: - install_failed - not_installed savedObject: + type: object + latestVersion: + type: string + keepPoliciesUpToDate: + type: boolean + notice: + type: string + licensePath: type: string required: - status diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index b2742b0d36886..b25266db5407c 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -57,7 +57,7 @@ export type DetailViewPanelName = | 'api-reference'; export type ServiceName = 'kibana' | 'elasticsearch'; export type AgentAssetType = typeof agentAssetTypes; -export type DocAssetType = 'doc' | 'notice'; +export type DocAssetType = 'doc' | 'notice' | 'license'; export type AssetType = | KibanaAssetType | ElasticsearchAssetType @@ -404,6 +404,7 @@ export interface EpmPackageAdditions { latestVersion: string; assets: AssetsGroupedByServiceByType; notice?: string; + licensePath?: string; keepPoliciesUpToDate?: boolean; } diff --git a/x-pack/plugins/fleet/common/types/models/package_spec.ts b/x-pack/plugins/fleet/common/types/models/package_spec.ts index 4d79b5167bf58..4463bb81097e2 100644 --- a/x-pack/plugins/fleet/common/types/models/package_spec.ts +++ b/x-pack/plugins/fleet/common/types/models/package_spec.ts @@ -15,6 +15,9 @@ export interface PackageSpecManifest { description: string; version: string; license?: 'basic'; + source?: { + license: string; + }; type?: 'integration' | 'input'; release?: 'experimental' | 'beta' | 'ga'; categories?: Array; @@ -54,12 +57,14 @@ export type PackageSpecCategory = | 'version_control' | 'web'; -export type PackageSpecConditions = Record< - 'kibana', - { +export interface PackageSpecConditions { + kibana: { version: string; - } ->; + }; + elastic?: { + subscription: string; + }; +} export interface PackageSpecIcon { src: string; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx index 162c9cb0d9efe..729975e8e8e37 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx @@ -33,6 +33,7 @@ import { useGetCategories } from '../../../../../hooks'; import { AssetTitleMap, DisplayedAssets, ServiceTitleMap } from '../../../constants'; import { NoticeModal } from './notice_modal'; +import { LicenseModal } from './license_modal'; const ReplacementCard = withSuspense(LazyReplacementCard); @@ -73,6 +74,11 @@ export const Details: React.FC = memo(({ packageInfo }) => { setIsNoticeModalOpen(!isNoticeModalOpen); }, [isNoticeModalOpen]); + const [isLicenseModalOpen, setIsLicenseModalOpen] = useState(false); + const toggleLicenseModal = useCallback(() => { + setIsLicenseModalOpen(!isLicenseModalOpen); + }, [isLicenseModalOpen]); + const listItems = useMemo(() => { // Base details: version and categories const items: EuiDescriptionListProps['listItems'] = [ @@ -154,8 +160,20 @@ export const Details: React.FC = memo(({ packageInfo }) => { }); } + // Subscription details + items.push({ + title: ( + + + + ), + description: ( +

{packageInfo.conditions?.elastic?.subscription || packageInfo.license || '-'}

+ ), + }); + // License details - if (packageInfo.license || packageInfo.notice) { + if (packageInfo.licensePath || packageInfo.source?.license || packageInfo.notice) { items.push({ title: ( @@ -164,7 +182,15 @@ export const Details: React.FC = memo(({ packageInfo }) => { ), description: ( <> -

{packageInfo.license}

+ {packageInfo.licensePath ? ( +

+ + {packageInfo.source?.license || 'LICENSE.txt'} + +

+ ) : ( +

{packageInfo.source?.license || '-'}

+ )} {packageInfo.notice && (

NOTICE.txt @@ -179,10 +205,14 @@ export const Details: React.FC = memo(({ packageInfo }) => { }, [ packageCategories, packageInfo.assets, + packageInfo.conditions?.elastic?.subscription, packageInfo.data_streams, packageInfo.license, + packageInfo.licensePath, packageInfo.notice, + packageInfo.source?.license, packageInfo.version, + toggleLicenseModal, toggleNoticeModal, ]); @@ -193,6 +223,15 @@ export const Details: React.FC = memo(({ packageInfo }) => { )} + + {isLicenseModalOpen && packageInfo.licensePath && ( + + )} + diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/license_modal.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/license_modal.tsx new file mode 100644 index 0000000000000..eafdbfb2debe2 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/license_modal.tsx @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useState } from 'react'; +import { + EuiCodeBlock, + EuiLoadingContent, + EuiModal, + EuiModalBody, + EuiModalHeader, + EuiModalFooter, + EuiModalHeaderTitle, + EuiButton, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { sendGetFileByPath, useStartServices } from '../../../../../hooks'; + +interface Props { + licenseName?: string; + licensePath: string; + onClose: () => void; +} + +export const LicenseModal: React.FunctionComponent = ({ + licenseName = 'LICENSE.txt', + licensePath, + onClose, +}) => { + const { notifications } = useStartServices(); + const [licenseText, setLicenseText] = useState(undefined); + + useEffect(() => { + async function fetchData() { + try { + const { data } = await sendGetFileByPath(licensePath); + setLicenseText(data || ''); + } catch (err) { + notifications.toasts.addError(err, { + title: i18n.translate('xpack.fleet.epm.errorLoadingLicense', { + defaultMessage: 'Error loading license information', + }), + }); + } + } + fetchData(); + }, [licensePath, notifications]); + return ( + + + +

{licenseName}

+ + + + + {licenseText ? ( + licenseText + ) : ( + // Simulate a long text while loading + <> +

+ +

+

+ +

+ + )} +
+
+ + + + + + + ); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index 4d70c62bd55d3..330839b13dba9 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -123,6 +123,13 @@ export function getPathParts(path: string): AssetParts { service = ''; } + // To support the LICENSE asset at the root level + if (service === 'LICENSE.txt') { + file = service; + type = 'license'; + service = ''; + } + // This is to cover for the fields.yml files inside the "fields" directory if (file === undefined) { file = type; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index dab8f3c0bf82e..527b93a66d130 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -223,7 +223,7 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName => name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX); -function buildComponentTemplates(params: { +export function buildComponentTemplates(params: { mappings: IndexTemplateMappings; templateName: string; registryElasticsearch: RegistryElasticsearch | undefined; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index ab8f60e172dcb..8c908ecc9ef87 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -7,65 +7,71 @@ import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { errors } from '@elastic/elasticsearch'; +import { safeLoad } from 'js-yaml'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { + PACKAGE_TEMPLATE_SUFFIX, + USER_SETTINGS_TEMPLATE_SUFFIX, +} from '../../../../../common/constants'; +import { + buildComponentTemplates, + installComponentAndIndexTemplateForDataStream, +} from '../template/install'; +import { processFields } from '../../fields/field'; +import { generateMappings } from '../template/template'; +import { getESAssetMetadata } from '../meta'; import { updateEsAssetReferences } from '../../packages/install'; import { getPathParts } from '../../archive'; import { ElasticsearchAssetType } from '../../../../../common/types/models'; -import type { EsAssetReference, InstallablePackage } from '../../../../../common/types/models'; +import type { + EsAssetReference, + InstallablePackage, + ESAssetMetadata, + IndexTemplate, + RegistryElasticsearch, +} from '../../../../../common/types/models'; import { getInstallation } from '../../packages'; - -import { getESAssetMetadata } from '../meta'; - import { retryTransientEsErrors } from '../retry'; import { deleteTransforms } from './remove'; import { getAsset } from './common'; -interface TransformInstallation { +const DEFAULT_TRANSFORM_TEMPLATES_PRIORITY = 250; +enum TRANSFORM_SPECS_TYPES { + MANIFEST = 'manifest', + FIELDS = 'fields', + TRANSFORM = 'transform', +} + +interface TransformModuleBase { + transformModuleId?: string; +} +interface DestinationIndexTemplateInstallation extends TransformModuleBase { + installationName: string; + _meta: ESAssetMetadata; + template: IndexTemplate['template']; +} +interface TransformInstallation extends TransformModuleBase { installationName: string; content: any; } -export const installTransform = async ( +const installLegacyTransformsAssets = async ( installablePackage: InstallablePackage, - paths: string[], + installNameSuffix: string, + transformPaths: string[], esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, logger: Logger, - esReferences?: EsAssetReference[] + esReferences: EsAssetReference[] = [], + previousInstalledTransformEsAssets: EsAssetReference[] = [] ) => { - const installation = await getInstallation({ - savedObjectsClient, - pkgName: installablePackage.name, - }); - esReferences = esReferences ?? installation?.installed_es ?? []; - let previousInstalledTransformEsAssets: EsAssetReference[] = []; - if (installation) { - previousInstalledTransformEsAssets = installation.installed_es.filter( - ({ type, id }) => type === ElasticsearchAssetType.transform - ); - if (previousInstalledTransformEsAssets.length) { - logger.info( - `Found previous transform references:\n ${JSON.stringify( - previousInstalledTransformEsAssets - )}` - ); - } - } - - // delete all previous transform - await deleteTransforms( - esClient, - previousInstalledTransformEsAssets.map((asset) => asset.id) - ); - - const installNameSuffix = `${installablePackage.version}`; - const transformPaths = paths.filter((path) => isTransform(path)); let installedTransforms: EsAssetReference[] = []; if (transformPaths.length > 0) { const transformRefs = transformPaths.reduce((acc, path) => { acc.push({ - id: getTransformNameForInstallation(installablePackage, path, installNameSuffix), + id: getLegacyTransformNameForInstallation(installablePackage, path, installNameSuffix), type: ElasticsearchAssetType.transform, }); @@ -87,7 +93,7 @@ export const installTransform = async ( content._meta = getESAssetMetadata({ packageName: installablePackage.name }); return { - installationName: getTransformNameForInstallation( + installationName: getLegacyTransformNameForInstallation( installablePackage, path, installNameSuffix @@ -117,6 +123,289 @@ export const installTransform = async ( return { installedTransforms, esReferences }; }; +const processTransformAssetsPerModule = ( + installablePackage: InstallablePackage, + installNameSuffix: string, + transformPaths: string[] +) => { + const transformsSpecifications = new Map(); + const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = []; + const transforms: TransformInstallation[] = []; + + transformPaths.forEach((path: string) => { + const { transformModuleId, fileName } = getTransformFolderAndFileNames( + installablePackage, + path + ); + + // Since there can be multiple assets per transform definition + // We want to create a unique list of assets/specifications for each transform + if (transformsSpecifications.get(transformModuleId) === undefined) { + transformsSpecifications.set(transformModuleId, new Map()); + } + const packageAssets = transformsSpecifications.get(transformModuleId); + + const content = safeLoad(getAsset(path).toString('utf-8')); + + if (fileName === TRANSFORM_SPECS_TYPES.FIELDS) { + const validFields = processFields(content); + const mappings = generateMappings(validFields); + packageAssets?.set('mappings', mappings); + } + + if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) { + transformsSpecifications.get(transformModuleId)?.set('destinationIndex', content.dest); + transformsSpecifications.get(transformModuleId)?.set('transform', content); + content._meta = getESAssetMetadata({ packageName: installablePackage.name }); + transforms.push({ + transformModuleId, + installationName: getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + `default-${installNameSuffix}` + ), + content, + }); + } + + if (fileName === TRANSFORM_SPECS_TYPES.MANIFEST) { + if (isPopulatedObject(content, ['start']) && content.start === false) { + transformsSpecifications.get(transformModuleId)?.set('start', false); + } + // If manifest.yml contains destination_index_template + // Combine the mappings and other index template settings from manifest.yml into a single index template + // Create the index template and track the template in EsAssetReferences + if ( + isPopulatedObject(content, ['destination_index_template']) || + isPopulatedObject(packageAssets.get('mappings')) + ) { + const destinationIndexTemplate = + (content.destination_index_template as Record) ?? {}; + destinationIndexTemplates.push({ + transformModuleId, + _meta: getESAssetMetadata({ packageName: installablePackage.name }), + installationName: getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ), + template: destinationIndexTemplate, + } as DestinationIndexTemplateInstallation); + packageAssets.set('destinationIndexTemplate', destinationIndexTemplate); + } + } + }); + + const indexTemplatesRefs = destinationIndexTemplates.map((template) => ({ + id: template.installationName, + type: ElasticsearchAssetType.indexTemplate, + })); + const componentTemplatesRefs = [ + ...destinationIndexTemplates.map((template) => ({ + id: `${template.installationName}${USER_SETTINGS_TEMPLATE_SUFFIX}`, + type: ElasticsearchAssetType.componentTemplate, + })), + ...destinationIndexTemplates.map((template) => ({ + id: `${template.installationName}${PACKAGE_TEMPLATE_SUFFIX}`, + type: ElasticsearchAssetType.componentTemplate, + })), + ]; + + const transformRefs = transforms.map((t) => ({ + id: t.installationName, + type: ElasticsearchAssetType.transform, + })); + + return { + indexTemplatesRefs, + componentTemplatesRefs, + transformRefs, + transforms, + destinationIndexTemplates, + transformsSpecifications, + }; +}; + +const installTransformsAssets = async ( + installablePackage: InstallablePackage, + installNameSuffix: string, + transformPaths: string[], + esClient: ElasticsearchClient, + savedObjectsClient: SavedObjectsClientContract, + logger: Logger, + esReferences: EsAssetReference[] = [], + previousInstalledTransformEsAssets: EsAssetReference[] = [] +) => { + let installedTransforms: EsAssetReference[] = []; + if (transformPaths.length > 0) { + const { + indexTemplatesRefs, + componentTemplatesRefs, + transformRefs, + transforms, + destinationIndexTemplates, + transformsSpecifications, + } = processTransformAssetsPerModule(installablePackage, installNameSuffix, transformPaths); + // get and save refs associated with the transforms before installing + esReferences = await updateEsAssetReferences( + savedObjectsClient, + installablePackage.name, + esReferences, + { + assetsToAdd: [...indexTemplatesRefs, ...componentTemplatesRefs, ...transformRefs], + assetsToRemove: previousInstalledTransformEsAssets, + } + ); + + // create index templates and component templates + await Promise.all( + destinationIndexTemplates + .map((destinationIndexTemplate) => { + const customMappings = transformsSpecifications + .get(destinationIndexTemplate.transformModuleId) + ?.get('mappings'); + const registryElasticsearch: RegistryElasticsearch = { + 'index_template.settings': destinationIndexTemplate.template.settings, + 'index_template.mappings': destinationIndexTemplate.template.mappings, + }; + + const componentTemplates = buildComponentTemplates({ + mappings: customMappings, + templateName: destinationIndexTemplate.installationName, + registryElasticsearch, + packageName: installablePackage.name, + defaultSettings: {}, + }); + + if (destinationIndexTemplate || customMappings) { + return installComponentAndIndexTemplateForDataStream({ + esClient, + logger, + componentTemplates, + indexTemplate: { + templateName: destinationIndexTemplate.installationName, + // @ts-expect-error We don't need to pass data_stream property here + // as this template is applied to only an index and not a data stream + indexTemplate: { + template: { settings: undefined, mappings: undefined }, + priority: DEFAULT_TRANSFORM_TEMPLATES_PRIORITY, + index_patterns: [ + transformsSpecifications + .get(destinationIndexTemplate.transformModuleId) + ?.get('destinationIndex').index, + ], + _meta: destinationIndexTemplate._meta, + composed_of: Object.keys(componentTemplates), + }, + }, + }); + } + }) + .filter((p) => p !== undefined) + ); + + // create destination indices + await Promise.all( + transforms.map(async (transform) => { + const index = transform.content.dest.index; + const pipelineId = transform.content.dest.pipeline; + + try { + await retryTransientEsErrors( + () => + esClient.indices.create( + { + index, + ...(pipelineId ? { settings: { default_pipeline: pipelineId } } : {}), + }, + { ignore: [400] } + ), + { logger } + ); + } catch (err) { + throw new Error(err.message); + } + }) + ); + + // create & optionally start transforms + const transformsPromises = transforms.map(async (transform) => { + return handleTransformInstall({ + esClient, + logger, + transform, + startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + }); + }); + + installedTransforms = await Promise.all(transformsPromises).then((results) => results.flat()); + } + + return { installedTransforms, esReferences }; +}; +export const installTransforms = async ( + installablePackage: InstallablePackage, + paths: string[], + esClient: ElasticsearchClient, + savedObjectsClient: SavedObjectsClientContract, + logger: Logger, + esReferences?: EsAssetReference[] +) => { + const transformPaths = paths.filter((path) => isTransform(path)); + + const installation = await getInstallation({ + savedObjectsClient, + pkgName: installablePackage.name, + }); + esReferences = esReferences ?? installation?.installed_es ?? []; + let previousInstalledTransformEsAssets: EsAssetReference[] = []; + if (installation) { + previousInstalledTransformEsAssets = installation.installed_es.filter( + ({ type, id }) => type === ElasticsearchAssetType.transform + ); + if (previousInstalledTransformEsAssets.length) { + logger.debug( + `Found previous transform references:\n ${JSON.stringify( + previousInstalledTransformEsAssets + )}` + ); + } + } + + // delete all previous transform + await deleteTransforms( + esClient, + previousInstalledTransformEsAssets.map((asset) => asset.id) + ); + + const installNameSuffix = `${installablePackage.version}`; + + // If package contains legacy transform specifications (i.e. with json instead of yml) + if (transformPaths.some((p) => p.endsWith('.json')) || transformPaths.length === 0) { + return await installLegacyTransformsAssets( + installablePackage, + installNameSuffix, + transformPaths, + esClient, + savedObjectsClient, + logger, + esReferences, + previousInstalledTransformEsAssets + ); + } + + return await installTransformsAssets( + installablePackage, + installNameSuffix, + transformPaths, + esClient, + savedObjectsClient, + logger, + esReferences, + previousInstalledTransformEsAssets + ); +}; + export const isTransform = (path: string) => { const pathParts = getPathParts(path); return !path.endsWith('/') && pathParts.type === ElasticsearchAssetType.transform; @@ -126,10 +415,12 @@ async function handleTransformInstall({ esClient, logger, transform, + startTransform, }: { esClient: ElasticsearchClient; logger: Logger; transform: TransformInstallation; + startTransform?: boolean; }): Promise { try { await retryTransientEsErrors( @@ -151,15 +442,21 @@ async function handleTransformInstall({ throw err; } } - await esClient.transform.startTransform( - { transform_id: transform.installationName }, - { ignore: [409] } - ); + + // start transform by default if not set in yml file + // else, respect the setting + if (startTransform === undefined || startTransform === true) { + await esClient.transform.startTransform( + { transform_id: transform.installationName }, + { ignore: [409] } + ); + logger.debug(`Started transform: ${transform.installationName}`); + } return { id: transform.installationName, type: ElasticsearchAssetType.transform }; } -const getTransformNameForInstallation = ( +const getLegacyTransformNameForInstallation = ( installablePackage: InstallablePackage, path: string, suffix: string @@ -169,3 +466,24 @@ const getTransformNameForInstallation = ( const folderName = pathPaths?.pop(); return `${installablePackage.name}.${folderName}-${filename}-${suffix}`; }; + +const getTransformAssetNameForInstallation = ( + installablePackage: InstallablePackage, + transformModuleId: string, + suffix?: string +) => { + return `logs-${installablePackage.name}.${transformModuleId}${suffix ? '-' + suffix : ''}`; +}; + +const getTransformFolderAndFileNames = (installablePackage: InstallablePackage, path: string) => { + const pathPaths = path.split('/'); + const fileName = pathPaths?.pop()?.split('.')[0]; + let transformModuleId = pathPaths?.pop(); + + // If fields.yml is located inside a directory called 'fields' (e.g. {exampleFolder}/fields/fields.yml) + // We need to go one level up to get the real folder name + if (transformModuleId === 'fields') { + transformModuleId = pathPaths?.pop(); + } + return { fileName: fileName ?? '', transformModuleId: transformModuleId ?? '' }; +}; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 1b8c347ad07d5..97fa1e94ca218 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -35,7 +35,7 @@ import { getESAssetMetadata } from '../meta'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants'; import { getAsset } from './common'; -import { installTransform } from './install'; +import { installTransforms } from './install'; describe('test transform install', () => { let esClient: ReturnType; @@ -122,7 +122,7 @@ describe('test transform install', () => { ], }); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', @@ -320,7 +320,7 @@ describe('test transform install', () => { } as unknown as SavedObject) ); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', @@ -422,7 +422,7 @@ describe('test transform install', () => { ], }); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', @@ -556,7 +556,7 @@ describe('test transform install', () => { ) ); - await installTransform( + await installTransforms( { name: 'endpoint', version: '0.16.0-dev.0', diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts index 1dc4039a7c2a7..4f98776f53d60 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts @@ -128,7 +128,7 @@ function getTest( test = { method: mocks.packageClient.reinstallEsAssets.bind(mocks.packageClient), args: [pkg, paths], - spy: jest.spyOn(epmTransformsInstall, 'installTransform'), + spy: jest.spyOn(epmTransformsInstall, 'installTransforms'), spyArgs: [pkg, paths, mocks.esClient, mocks.soClient, mocks.logger], spyResponse: { installedTransforms: [ diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index c43386acdbdf4..71ab87c7d92bd 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -24,7 +24,7 @@ import type { import { checkSuperuser } from '../../routes/security'; import { FleetUnauthorizedError } from '../../errors'; -import { installTransform, isTransform } from './elasticsearch/transform/install'; +import { installTransforms, isTransform } from './elasticsearch/transform/install'; import { fetchFindLatestPackageOrThrow, getRegistryPackage } from './registry'; import { ensureInstalledPackage, getInstallation } from './packages'; @@ -151,7 +151,7 @@ class PackageClientImpl implements PackageClient { } async #reinstallTransforms(packageInfo: InstallablePackage, paths: string[]) { - const { installedTransforms } = await installTransform( + const { installedTransforms } = await installTransforms( packageInfo, paths, this.internalEsClient, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 8168bb05e53a4..4ecec17560731 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -42,7 +42,7 @@ import { import { installILMPolicy } from '../elasticsearch/ilm/install'; import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; -import { installTransform } from '../elasticsearch/transform/install'; +import { installTransforms } from '../elasticsearch/transform/install'; import { installMlModel } from '../elasticsearch/ml_model'; import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; import { saveArchiveEntries } from '../archive/storage'; @@ -219,7 +219,7 @@ export async function _installPackage({ ); ({ esReferences } = await withPackageSpan('Install transforms', () => - installTransform(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) + installTransforms(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) )); // If this is an update or retrying an update, delete the previous version's pipelines diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index e69c5fe3944f0..c333ba9f7bc3d 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -181,6 +181,7 @@ export async function getPackageInfo({ title: packageInfo.title || nameAsTitle(packageInfo.name), assets: Registry.groupPathsByService(paths || []), notice: Registry.getNoticePath(paths || []), + licensePath: Registry.getLicensePath(paths || []), keepPoliciesUpToDate: savedObject?.attributes.keep_policies_up_to_date ?? false, }; const updated = { ...packageInfo, ...additions }; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts index e9427732d4aef..ba309a15b04ee 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts @@ -9,7 +9,12 @@ import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { PackageNotFoundError } from '../../../errors'; -import { splitPkgKey, fetchFindLatestPackageOrUndefined, fetchFindLatestPackageOrThrow } from '.'; +import { + splitPkgKey, + fetchFindLatestPackageOrUndefined, + fetchFindLatestPackageOrThrow, + getLicensePath, +} from '.'; const mockLoggerFactory = loggingSystemMock.create(); const mockLogger = mockLoggerFactory.get('mock logger'); @@ -143,3 +148,26 @@ describe('fetch package', () => { }); }); }); + +describe('getLicensePath', () => { + it('returns first license path if found', () => { + const path = getLicensePath([ + '/package/good-1.0.0/NOTICE.txt', + '/package/good-1.0.0/changelog.yml', + '/package/good-1.0.0/manifest.yml', + '/package/good-1.0.0/LICENSE.txt', + '/package/good-1.0.0/docs/README.md', + ]); + expect(path).toEqual('/package/good/1.0.0/LICENSE.txt'); + }); + + it('returns undefined if no license', () => { + const path = getLicensePath([ + '/package/good-1.0.0/NOTICE.txt', + '/package/good-1.0.0/changelog.yml', + '/package/good-1.0.0/manifest.yml', + '/package/good-1.0.0/docs/README.md', + ]); + expect(path).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 0302c0188cc2c..fee603f188e29 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -392,3 +392,15 @@ export function getNoticePath(paths: string[]): string | undefined { return undefined; } + +export function getLicensePath(paths: string[]): string | undefined { + for (const path of paths) { + const parts = getPathParts(path.replace(/^\/package\//, '')); + if (parts.type === 'license') { + const { pkgName, pkgVersion } = splitPkgKey(parts.pkgkey); + return `/package/${pkgName}/${pkgVersion}/${parts.file}`; + } + } + + return undefined; +} diff --git a/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts b/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts index 6f823f218ca23..3642448774c58 100644 --- a/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts +++ b/x-pack/plugins/maps/public/embeddable/map_embeddable_factory.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { first } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { EmbeddableFactoryDefinition, IContainer } from '@kbn/embeddable-plugin/public'; import { MAP_SAVED_OBJECT_TYPE, APP_ICON } from '../../common/constants'; @@ -12,6 +13,7 @@ import { getMapEmbeddableDisplayName } from '../../common/i18n_getters'; import { extract, inject } from '../../common/embeddable'; import { MapByReferenceInput, MapEmbeddableInput } from './types'; import { lazyLoadMapModules } from '../lazy_load_bundle'; +import { getApplication, getUsageCollection } from '../kibana_services'; export class MapEmbeddableFactory implements EmbeddableFactoryDefinition { type = MAP_SAVED_OBJECT_TYPE; @@ -50,6 +52,15 @@ export class MapEmbeddableFactory implements EmbeddableFactoryDefinition { create = async (input: MapEmbeddableInput, parent?: IContainer) => { const { MapEmbeddable } = await lazyLoadMapModules(); + const usageCollection = getUsageCollection(); + if (usageCollection) { + // currentAppId$ is a BehaviorSubject exposed as an observable so subscription gets last value upon subscribe + getApplication() + .currentAppId$.pipe(first()) + .subscribe((appId) => { + if (appId) usageCollection.reportUiCounter('map', 'loaded', `open_maps_vis_${appId}`); + }); + } return new MapEmbeddable( { editable: await this.isEditable(), diff --git a/x-pack/plugins/maps/public/kibana_services.ts b/x-pack/plugins/maps/public/kibana_services.ts index 41f88de18720c..a1a8a19674a81 100644 --- a/x-pack/plugins/maps/public/kibana_services.ts +++ b/x-pack/plugins/maps/public/kibana_services.ts @@ -66,6 +66,7 @@ export const getPresentationUtilContext = () => pluginsStart.presentationUtil.Co export const getSecurityService = () => pluginsStart.security; export const getSpacesApi = () => pluginsStart.spaces; export const getTheme = () => coreStart.theme; +export const getApplication = () => coreStart.application; export const getUsageCollection = () => pluginsStart.usageCollection; export const isScreenshotMode = () => { return pluginsStart.screenshotMode ? pluginsStart.screenshotMode.isScreenshotMode() : false; diff --git a/x-pack/plugins/observability/server/assets/constants.ts b/x-pack/plugins/observability/server/assets/constants.ts index ea96f7a3b5763..4c0cc0e2e6f83 100644 --- a/x-pack/plugins/observability/server/assets/constants.ts +++ b/x-pack/plugins/observability/server/assets/constants.ts @@ -5,12 +5,14 @@ * 2.0. */ -export const SLO_COMPONENT_TEMPLATE_MAPPINGS_NAME = 'observability-slo-mappings'; -export const SLO_COMPONENT_TEMPLATE_SETTINGS_NAME = 'observability-slo-settings'; +export const SLO_COMPONENT_TEMPLATE_MAPPINGS_NAME = 'slo-observability.sli-mappings'; +export const SLO_COMPONENT_TEMPLATE_SETTINGS_NAME = 'slo-observability.sli-settings'; export const SLO_INDEX_TEMPLATE_NAME = 'slo-observability.sli'; -export const SLO_INGEST_PIPELINE_NAME = 'observability-slo-monthly-index'; export const SLO_RESOURCES_VERSION = 1; +export const getSLOIngestPipelineName = (spaceId: string) => + `${SLO_INDEX_TEMPLATE_NAME}.monthly-${spaceId}`; + export const getSLODestinationIndexName = (spaceId: string) => `${SLO_INDEX_TEMPLATE_NAME}-v${SLO_RESOURCES_VERSION}-${spaceId}`; diff --git a/x-pack/plugins/observability/server/routes/register_routes.ts b/x-pack/plugins/observability/server/routes/register_routes.ts index 1a1fdf220afb2..e0e9e94f5cb21 100644 --- a/x-pack/plugins/observability/server/routes/register_routes.ts +++ b/x-pack/plugins/observability/server/routes/register_routes.ts @@ -70,6 +70,10 @@ export function registerRoutes({ spacesService, })) as any; + if (data === undefined) { + return response.noContent(); + } + return response.ok({ body: data }); } catch (error) { if (error instanceof ObservabilityError) { diff --git a/x-pack/plugins/observability/server/saved_objects/slo.ts b/x-pack/plugins/observability/server/saved_objects/slo.ts index 13b418e5442c0..59a00cb30a558 100644 --- a/x-pack/plugins/observability/server/saved_objects/slo.ts +++ b/x-pack/plugins/observability/server/saved_objects/slo.ts @@ -39,11 +39,6 @@ export const slo: SavedObjectsType = { target: { type: 'float' }, }, }, - settings: { - properties: { - destination_index: { type: 'text' }, - }, - }, created_at: { type: 'date' }, updated_at: { type: 'date' }, }, diff --git a/x-pack/plugins/observability/server/services/slo/create_slo.ts b/x-pack/plugins/observability/server/services/slo/create_slo.ts index c90bd9c5c2ccc..d04e7013a38f9 100644 --- a/x-pack/plugins/observability/server/services/slo/create_slo.ts +++ b/x-pack/plugins/observability/server/services/slo/create_slo.ts @@ -52,9 +52,6 @@ export class CreateSLO { return { ...sloParams, id: uuid.v1(), - settings: { - destination_index: sloParams.settings?.destination_index, - }, }; } diff --git a/x-pack/plugins/observability/server/services/slo/delete_slo.test.ts b/x-pack/plugins/observability/server/services/slo/delete_slo.test.ts index 996928eae34b9..b1a68b9c04aee 100644 --- a/x-pack/plugins/observability/server/services/slo/delete_slo.test.ts +++ b/x-pack/plugins/observability/server/services/slo/delete_slo.test.ts @@ -7,7 +7,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; -import { getSLOTransformId } from '../../assets/constants'; +import { getSLOTransformId, SLO_INDEX_TEMPLATE_NAME } from '../../assets/constants'; import { DeleteSLO } from './delete_slo'; import { createAPMTransactionErrorRateIndicator, createSLO } from './fixtures/slo'; import { createSLORepositoryMock, createTransformManagerMock } from './mocks'; @@ -30,7 +30,6 @@ describe('DeleteSLO', () => { describe('happy path', () => { it('removes the transform, the roll up data and the SLO from the repository', async () => { const slo = createSLO(createAPMTransactionErrorRateIndicator()); - mockRepository.findById.mockResolvedValueOnce(slo); await deleteSLO.execute(slo.id); @@ -38,6 +37,7 @@ describe('DeleteSLO', () => { expect(mockTransformManager.uninstall).toHaveBeenCalledWith(getSLOTransformId(slo.id)); expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith( expect.objectContaining({ + index: `${SLO_INDEX_TEMPLATE_NAME}*`, query: { match: { 'slo.id': slo.id, diff --git a/x-pack/plugins/observability/server/services/slo/delete_slo.ts b/x-pack/plugins/observability/server/services/slo/delete_slo.ts index 5c56f47efd66e..59e5df8a5975a 100644 --- a/x-pack/plugins/observability/server/services/slo/delete_slo.ts +++ b/x-pack/plugins/observability/server/services/slo/delete_slo.ts @@ -7,7 +7,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { getSLOTransformId, SLO_INDEX_TEMPLATE_NAME } from '../../assets/constants'; -import { SLO } from '../../types/models'; + import { SLORepository } from './slo_repository'; import { TransformManager } from './transform_manager'; @@ -19,23 +19,21 @@ export class DeleteSLO { ) {} public async execute(sloId: string): Promise { - const slo = await this.repository.findById(sloId); - const sloTransformId = getSLOTransformId(sloId); await this.transformManager.stop(sloTransformId); await this.transformManager.uninstall(sloTransformId); - await this.deleteRollupData(slo); + await this.deleteRollupData(sloId); await this.repository.deleteById(sloId); } - private async deleteRollupData(slo: SLO): Promise { + private async deleteRollupData(sloId: string): Promise { await this.esClient.deleteByQuery({ - index: slo.settings.destination_index ?? `${SLO_INDEX_TEMPLATE_NAME}*`, + index: `${SLO_INDEX_TEMPLATE_NAME}*`, wait_for_completion: false, query: { match: { - 'slo.id': slo.id, + 'slo.id': sloId, }, }, }); diff --git a/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts b/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts index 536c9f01c31c1..41ea0ec67991d 100644 --- a/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts +++ b/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts @@ -30,9 +30,6 @@ export const createSLOParams = (indicator: SLI): CreateSLOParams => ({ export const createSLO = (indicator: SLI): SLO => ({ ...commonSLO, id: uuid.v1(), - settings: { - destination_index: 'some-index', - }, indicator, }); diff --git a/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts b/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts index 40dc0e46022bc..f8746f38cd246 100644 --- a/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts +++ b/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts @@ -9,14 +9,15 @@ import { IngestGetPipelineResponse } from '@elastic/elasticsearch/lib/api/typesW import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { loggerMock } from '@kbn/logging-mocks'; import { + getSLOIngestPipelineName, SLO_COMPONENT_TEMPLATE_MAPPINGS_NAME, SLO_COMPONENT_TEMPLATE_SETTINGS_NAME, SLO_INDEX_TEMPLATE_NAME, - SLO_INGEST_PIPELINE_NAME, SLO_RESOURCES_VERSION, } from '../../assets/constants'; import { DefaultResourceInstaller } from './resource_installer'; +const SPACE_ID = 'space-id'; describe('resourceInstaller', () => { describe("when the common resources don't exist", () => { it('installs the common resources', async () => { @@ -25,7 +26,7 @@ describe('resourceInstaller', () => { const installer = new DefaultResourceInstaller( mockClusterClient, loggerMock.create(), - 'space-id' + SPACE_ID ); await installer.ensureCommonResourcesInstalled(); @@ -43,7 +44,7 @@ describe('resourceInstaller', () => { expect.objectContaining({ name: SLO_INDEX_TEMPLATE_NAME }) ); expect(mockClusterClient.ingest.putPipeline).toHaveBeenCalledWith( - expect.objectContaining({ id: SLO_INGEST_PIPELINE_NAME }) + expect.objectContaining({ id: getSLOIngestPipelineName(SPACE_ID) }) ); }); }); @@ -53,12 +54,13 @@ describe('resourceInstaller', () => { const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); mockClusterClient.indices.existsIndexTemplate.mockResponseOnce(true); mockClusterClient.ingest.getPipeline.mockResponseOnce({ - [SLO_INGEST_PIPELINE_NAME]: { _meta: { version: SLO_RESOURCES_VERSION } }, + // @ts-ignore _meta not typed properly + [getSLOIngestPipelineName(SPACE_ID)]: { _meta: { version: SLO_RESOURCES_VERSION } }, } as IngestGetPipelineResponse); const installer = new DefaultResourceInstaller( mockClusterClient, loggerMock.create(), - 'space-id' + SPACE_ID ); await installer.ensureCommonResourcesInstalled(); diff --git a/x-pack/plugins/observability/server/services/slo/resource_installer.ts b/x-pack/plugins/observability/server/services/slo/resource_installer.ts index 280183d70459a..8ed8108b3c4b7 100644 --- a/x-pack/plugins/observability/server/services/slo/resource_installer.ts +++ b/x-pack/plugins/observability/server/services/slo/resource_installer.ts @@ -13,10 +13,10 @@ import type { import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { + getSLOIngestPipelineName, SLO_COMPONENT_TEMPLATE_MAPPINGS_NAME, SLO_COMPONENT_TEMPLATE_SETTINGS_NAME, SLO_INDEX_TEMPLATE_NAME, - SLO_INGEST_PIPELINE_NAME, SLO_RESOURCES_VERSION, } from '../../assets/constants'; import { getSLOMappingsTemplate } from '../../assets/component_templates/slo_mappings_template'; @@ -64,7 +64,7 @@ export class DefaultResourceInstaller implements ResourceInstaller { await this.createOrUpdateIngestPipelineTemplate( getSLOPipelineTemplate( - SLO_INGEST_PIPELINE_NAME, + getSLOIngestPipelineName(this.spaceId), this.getPipelinePrefix(SLO_RESOURCES_VERSION, this.spaceId) ) ); @@ -87,13 +87,12 @@ export class DefaultResourceInstaller implements ResourceInstaller { let ingestPipelineExists = false; try { - const pipeline = await this.esClient.ingest.getPipeline({ - id: SLO_INGEST_PIPELINE_NAME, - }); + const pipelineName = getSLOIngestPipelineName(this.spaceId); + const pipeline = await this.esClient.ingest.getPipeline({ id: pipelineName }); ingestPipelineExists = // @ts-ignore _meta is not defined on the type - pipeline && pipeline[SLO_INGEST_PIPELINE_NAME]._meta.version === SLO_RESOURCES_VERSION; + pipeline && pipeline[pipelineName]._meta.version === SLO_RESOURCES_VERSION; } catch (err) { return false; } diff --git a/x-pack/plugins/observability/server/services/slo/slo_repository.ts b/x-pack/plugins/observability/server/services/slo/slo_repository.ts index ccb6b63b048ed..29e16346fcdb1 100644 --- a/x-pack/plugins/observability/server/services/slo/slo_repository.ts +++ b/x-pack/plugins/observability/server/services/slo/slo_repository.ts @@ -69,6 +69,5 @@ function toSLOModel(slo: StoredSLO): SLO { time_window: slo.time_window, budgeting_method: slo.budgeting_method, objective: slo.objective, - settings: slo.settings, }; } diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap index ade6f8b90d894..29ce4877b6b34 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap @@ -20,7 +20,8 @@ Object { "version": 1, }, "dest": Object { - "index": "some-index", + "index": "slo-observability.sli-v1-my-namespace", + "pipeline": "slo-observability.sli.monthly-my-namespace", }, "frequency": "1m", "pivot": Object { diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap index d07a06e0724cf..58ed238fcf0a7 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap @@ -20,7 +20,8 @@ Object { "version": 1, }, "dest": Object { - "index": "some-index", + "index": "slo-observability.sli-v1-my-namespace", + "pipeline": "slo-observability.sli.monthly-my-namespace", }, "frequency": "1m", "pivot": Object { diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts index 274abe86e3a52..9c0a5aafc693f 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts @@ -12,8 +12,8 @@ import { } from '@elastic/elasticsearch/lib/api/types'; import { getSLODestinationIndexName, + getSLOIngestPipelineName, getSLOTransformId, - SLO_INGEST_PIPELINE_NAME, } from '../../../assets/constants'; import { getSLOTransformTemplate } from '../../../assets/transform_templates/slo_transform_template'; import { @@ -105,14 +105,10 @@ export class ApmTransactionDurationTransformGenerator implements TransformGenera } private buildDestination(slo: APMTransactionDurationSLO, spaceId: string) { - if (slo.settings.destination_index === undefined) { - return { - pipeline: SLO_INGEST_PIPELINE_NAME, - index: getSLODestinationIndexName(spaceId), - }; - } - - return { index: slo.settings.destination_index }; + return { + pipeline: getSLOIngestPipelineName(spaceId), + index: getSLODestinationIndexName(spaceId), + }; } private buildGroupBy() { diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts index da65bca20392e..6fa806e72e197 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts @@ -14,8 +14,8 @@ import { getSLOTransformTemplate } from '../../../assets/transform_templates/slo import { TransformGenerator } from '.'; import { getSLODestinationIndexName, + getSLOIngestPipelineName, getSLOTransformId, - SLO_INGEST_PIPELINE_NAME, } from '../../../assets/constants'; import { apmTransactionErrorRateSLOSchema, @@ -107,14 +107,10 @@ export class ApmTransactionErrorRateTransformGenerator implements TransformGener } private buildDestination(slo: APMTransactionErrorRateSLO, spaceId: string) { - if (slo.settings.destination_index === undefined) { - return { - pipeline: SLO_INGEST_PIPELINE_NAME, - index: getSLODestinationIndexName(spaceId), - }; - } - - return { index: slo.settings.destination_index }; + return { + pipeline: getSLOIngestPipelineName(spaceId), + index: getSLODestinationIndexName(spaceId), + }; } private buildGroupBy() { diff --git a/x-pack/plugins/observability/server/types/models/slo.ts b/x-pack/plugins/observability/server/types/models/slo.ts index 0cbb60531cc36..7420255b28974 100644 --- a/x-pack/plugins/observability/server/types/models/slo.ts +++ b/x-pack/plugins/observability/server/types/models/slo.ts @@ -25,9 +25,6 @@ const baseSLOSchema = t.type({ objective: t.type({ target: t.number, }), - settings: t.partial({ - destination_index: t.string, - }), }); export const apmTransactionErrorRateSLOSchema = t.intersection([ diff --git a/x-pack/plugins/observability/server/types/schema/slo.ts b/x-pack/plugins/observability/server/types/schema/slo.ts index c601b49270f89..3d1d3e93bc859 100644 --- a/x-pack/plugins/observability/server/types/schema/slo.ts +++ b/x-pack/plugins/observability/server/types/schema/slo.ts @@ -55,26 +55,17 @@ export const indicatorSchema = t.union([ apmTransactionErrorRateIndicatorSchema, ]); -const sloOptionalSettingsSchema = t.partial({ - settings: t.partial({ - destination_index: t.string, +const createSLOBodySchema = t.type({ + name: t.string, + description: t.string, + indicator: indicatorSchema, + time_window: rollingTimeWindowSchema, + budgeting_method: t.literal('occurrences'), + objective: t.type({ + target: t.number, }), }); -const createSLOBodySchema = t.intersection([ - t.type({ - name: t.string, - description: t.string, - indicator: indicatorSchema, - time_window: rollingTimeWindowSchema, - budgeting_method: t.literal('occurrences'), - objective: t.type({ - target: t.number, - }), - }), - sloOptionalSettingsSchema, -]); - const createSLOResponseSchema = t.type({ id: t.string, }); diff --git a/x-pack/plugins/security_solution/public/management/pages/host_isolation_exceptions/view/components/form.test.tsx b/x-pack/plugins/security_solution/public/management/pages/host_isolation_exceptions/view/components/form.test.tsx index 23f8ea83a7094..4c7ac15a504a7 100644 --- a/x-pack/plugins/security_solution/public/management/pages/host_isolation_exceptions/view/components/form.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/host_isolation_exceptions/view/components/form.test.tsx @@ -29,7 +29,8 @@ import type { HttpFetchOptionsWithPath } from '@kbn/core/public'; jest.mock('../../../../../common/components/user_privileges'); -describe('When on the host isolation exceptions entry form', () => { +// FLAKY: https://github.com/elastic/kibana/issues/140907 +describe.skip('When on the host isolation exceptions entry form', () => { let render: () => Promise>; let renderResult: ReturnType; let history: AppContextTestRender['history']; diff --git a/x-pack/plugins/session_view/public/components/session_view/index.tsx b/x-pack/plugins/session_view/public/components/session_view/index.tsx index 01f072f5c52b5..07626d090bd2b 100644 --- a/x-pack/plugins/session_view/public/components/session_view/index.tsx +++ b/x-pack/plugins/session_view/public/components/session_view/index.tsx @@ -13,7 +13,6 @@ import { EuiPanel, EuiHorizontalRule, EuiFlexGroup, - EuiBetaBadge, EuiButtonIcon, EuiToolTip, } from '@elastic/eui'; @@ -40,7 +39,7 @@ import { useFetchGetTotalIOBytes, } from './hooks'; import { LOCAL_STORAGE_DISPLAY_OPTIONS_KEY } from '../../../common/constants'; -import { BETA, REFRESH_SESSION, TOGGLE_TTY_PLAYER, DETAIL_PANEL } from './translations'; +import { REFRESH_SESSION, TOGGLE_TTY_PLAYER, DETAIL_PANEL } from './translations'; /** * The main wrapper component for the session view. @@ -259,9 +258,6 @@ export const SessionView = ({
- - - { const { euiTheme, euiVars } = useEuiTheme(); const cached = useMemo(() => { - const { border, colors, size } = euiTheme; + const { border, size } = euiTheme; // 118px = Session View Toolbar height + Close Session button height + spacing margin at the bottom const sessionView: CSSObject = { @@ -55,17 +55,12 @@ export const useStyles = ({ height = 500, isFullScreen }: StylesDeps) => { }, }; - const betaBadge: CSSObject = { - backgroundColor: `${colors.emptyShade}`, - }; - return { processTree, detailPanel, nonGrowGroup, resizeHandle, sessionViewerComponent, - betaBadge, }; }, [euiTheme, isFullScreen, height, euiVars]); diff --git a/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts b/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts index ccd91253012b3..f9f1252f4765e 100644 --- a/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts +++ b/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts @@ -25,6 +25,9 @@ import { ENDING_BREADCRUMB, FIELD_BROWSER, FIELD_BROWSER_MODAL, + FIELD_SELECTOR_TOGGLE_BUTTON, + FIELD_SELECTOR_INPUT, + FIELD_SELECTOR_LIST, } from '../screens/indicators'; import { login } from '../tasks/login'; import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver'; @@ -143,16 +146,13 @@ describe('Indicators', () => { it('should have the default selected field, then update when user selects', () => { const threatFeedName = 'threat.feed.name'; - cy.get(`${FIELD_SELECTOR}`).should('have.value', threatFeedName); + cy.get(`${FIELD_SELECTOR_INPUT}`).eq(0).should('have.text', threatFeedName); const threatIndicatorIp: string = 'threat.indicator.ip'; - cy.get(`${FIELD_SELECTOR}`) - .should('exist') - .select(threatIndicatorIp) - .should('have.value', threatIndicatorIp); + cy.get(`${FIELD_SELECTOR_TOGGLE_BUTTON}`).should('exist').click(); - cy.get(`${FIELD_SELECTOR}`).should('have.value', threatIndicatorIp); + cy.get(`${FIELD_SELECTOR_LIST}`).should('exist').contains(threatIndicatorIp); }); }); diff --git a/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts b/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts index 4ceaa0c020e99..b5582da6ce8ef 100644 --- a/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts +++ b/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts @@ -41,6 +41,12 @@ export const INDICATOR_TYPE_CELL = '[data-gridcell-column-id="threat.indicator.t export const FIELD_SELECTOR = '[data-test-subj="tiIndicatorFieldSelectorDropdown"]'; +export const FIELD_SELECTOR_INPUT = '[data-test-subj="comboBoxInput"]'; + +export const FIELD_SELECTOR_TOGGLE_BUTTON = '[data-test-subj="comboBoxToggleListButton"]'; + +export const FIELD_SELECTOR_LIST = '[data-test-subj="comboBoxOptionsList"]'; + export const FIELD_BROWSER = `[data-test-subj="show-field-browser"]`; export const FIELD_BROWSER_MODAL = `[data-test-subj="fields-browser-container"]`; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap index de4d6c198961f..bc2b71303138b 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap @@ -21,43 +21,67 @@ Object { class="euiFlexItem euiFlexItem--flexGrowZero" >
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
@@ -102,43 +126,67 @@ Object { class="euiFlexItem euiFlexItem--flexGrowZero" >
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap index 735a1c34718c0..9346a739b9dae 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap @@ -6,43 +6,67 @@ Object { "baseElement":
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
@@ -50,43 +74,67 @@ Object { , "container":
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
@@ -151,32 +199,67 @@ Object { "baseElement":
-
- +
+
+
+
+ +
@@ -184,32 +267,67 @@ Object { , "container":
-
- +
+
+
+
+ +
diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx index e3bd4129d4fc5..0d62bbcef6219 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx @@ -9,7 +9,7 @@ import React from 'react'; import { render } from '@testing-library/react'; import { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { TestProvidersComponent } from '../../../../common/mocks/test_providers'; -import { DROPDOWN_TEST_ID, IndicatorsFieldSelector } from './indicators_field_selector'; +import { IndicatorsFieldSelector } from './indicators_field_selector'; const mockIndexPattern: DataView = { fields: [ @@ -51,11 +51,5 @@ describe('', () => { ); expect(component).toMatchSnapshot(); - - const dropdownOptions: string = component.getByTestId(DROPDOWN_TEST_ID).innerHTML; - const optionsCount: number = (dropdownOptions.match(/