From 3ece9501560ae18bb6c5993db7cc889a6675857c Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Fri, 25 Oct 2024 11:20:26 +0200 Subject: [PATCH] [Dataset Quality] Add fix it flow for field limit (#195561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Closes - https://github.com/elastic/kibana/issues/190330 This PR implements the logic to support - One click increasing of Field Limit for Field Limit Issues (applicable on for Integrations). For Non Integrations, only text is displayed as to how they can do it. - The One click increase updates the linked custom component template as well as the last backing Index - If Last Backing Index update fails due to any reason, it provides user an option to trigger a Rollover manually. ## Demo Not possible, to many things to display 😆 ## What's Pending ? Tests - [x] API tests - [x] Settings API - [x] Rollover API - [x] Apply New limit API - [x] FTR tests - [x] Displaying of various issues for integrations and non integrations - [x] Fix it Flow Good case, without Rollover - [x] Fix it Flow Good case, with Rollover - [x] Manual Mitigation - Click on Component Template shold navigate to proper logic based on Integration / Non - [x] Manual Mitigation - Ingest Pipeline - [x] Link for official Documentation ## How to setup a local environment We will be setting up 2 different data streams, one with integration and one without. Please follow the steps in the exact order 1. Start Local ES and Local Kibana 2. Install Nginx Integration 1st 3. Ingest data as per script here - https://gist.github.com/achyutjhunjhunwala/03ea29190c6594544f584d2f0efa71e5 4. Set the Limit for the 2 datasets ``` PUT logs-synth.3-default/_settings { "mapping.total_fields.limit": 36 } // Set the limit for Nginx PUT logs-nginx.access-default/_settings { "mapping.total_fields.limit": 52 } ``` 5. Now uncomment line number 59 from the synthtrace script to enable cloud.project.id field and run the scenario again 6. Do a Rollover ``` POST logs-synth.3-default/_rollover POST logs-nginx.access-default/_rollover ``` 7. Get last backing index for both dataset ``` GET _data_stream/logs-synth.3-default/ GET _data_stream/logs-nginx.access-default ``` 8. Increase the Limit by 1 but for last backing index ``` PUT .ds-logs-synth.3-default-2024.10.10-000002/_settings { "mapping.total_fields.limit": 37 } PUT .ds-logs-nginx.access-default-2024.10.10-000002/_settings { "mapping.total_fields.limit": 53 } ``` 9. Run the same Synthtrace scenario again. This setup will give you 3 fields for testings 1. cloud.availability_zone - Which will show the character limit isue 2. cloud.project - Which will show an obsolete error which happened in the past and now does not exists due to field limit 3. cloud.project.id - A current field limit issue --------- Co-authored-by: Marco Antonio Ghiani Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../src/types.ts | 19 +- .../public/application/services/routing.ts | 6 + .../index_management/public/locator.test.ts | 22 + .../index_management/public/locator.ts | 18 +- .../dataset_quality/common/api_types.ts | 27 +- .../common/data_stream_details/types.ts | 5 + .../dataset_quality/common/translations.ts | 179 +++++ .../common/utils/component_template_name.ts | 18 + .../degraded_field_flyout/field_info.tsx | 98 +-- .../degraded_field_flyout/index.tsx | 44 +- .../field_limit_documentation_link.tsx | 28 + .../field_limit/field_mapping_limit.tsx | 83 ++ .../increase_field_mapping_limit.tsx | 83 ++ .../field_limit/message_callout.tsx | 119 +++ .../possible_mitigations/index.tsx | 35 + .../manual/component_template_link.tsx | 86 +++ .../possible_mitigations/manual/index.tsx | 39 + .../manual/pipeline_link.tsx | 136 ++++ .../possible_mitigations/title.tsx | 35 + .../degraded_fields/degraded_fields.tsx | 1 + .../public/hooks/use_degraded_fields.ts | 80 +- .../data_stream_details_client.ts | 47 ++ .../services/data_stream_details/types.ts | 11 +- .../notifications.ts | 25 + .../state_machine.ts | 186 ++++- .../types.ts | 82 +- .../get_data_stream_details/index.ts | 4 +- .../get_datastream_settings.ts | 19 +- .../get_degraded_field_analysis/index.ts | 9 +- .../server/routes/data_streams/routes.ts | 58 ++ .../data_streams/update_field_limit/index.ts | 59 ++ .../update_component_template.ts | 53 ++ .../update_settings_last_backing_index.ts | 41 + .../utils/create_dataset_quality_es_client.ts | 28 +- .../dataset_quality/data_stream_rollover.ts | 86 +++ .../dataset_quality/data_stream_settings.ts | 251 ++++++ .../dataset_quality/degraded_field_analyze.ts | 2 +- .../observability/dataset_quality/index.ts | 3 + .../dataset_quality/integrations.ts | 33 +- .../dataset_quality/update_field_limit.ts | 176 +++++ .../dataset_quality/{ => utils}/es_utils.ts | 18 + .../data_streams/data_stream_settings.spec.ts | 159 ---- .../custom_integration_mappings.ts | 177 +++++ .../dataset_quality/degraded_field_flyout.ts | 691 +++++++++++++---- .../page_objects/dataset_quality.ts | 25 + .../data_stream_settings.ts | 130 ---- .../dataset_quality_api_integration/index.ts | 1 - .../custom_integration_mappings.ts | 177 +++++ .../dataset_quality_details.ts | 2 + .../dataset_quality/degraded_field_flyout.ts | 712 ++++++++++++++---- 50 files changed, 3707 insertions(+), 719 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts rename x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/{ => utils}/es_utils.ts (63%) delete mode 100644 x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts create mode 100644 x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts create mode 100644 x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts diff --git a/x-pack/packages/index-management/index_management_shared_types/src/types.ts b/x-pack/packages/index-management/index_management_shared_types/src/types.ts index 190aadfc29d01..ec5c7938d6b4b 100644 --- a/x-pack/packages/index-management/index_management_shared_types/src/types.ts +++ b/x-pack/packages/index-management/index_management_shared_types/src/types.ts @@ -17,10 +17,21 @@ import type { LocatorPublic } from '@kbn/share-plugin/public'; import { ExtensionsSetup } from './services/extensions_service'; import { PublicApiServiceSetup } from './services/public_api_service'; -export interface IndexManagementLocatorParams extends SerializableRecord { - page: 'data_streams_details'; - dataStreamName?: string; -} +export type IndexManagementLocatorParams = SerializableRecord & + ( + | { + page: 'data_streams_details'; + dataStreamName?: string; + } + | { + page: 'index_template'; + indexTemplate: string; + } + | { + page: 'component_template'; + componentTemplate: string; + } + ); export type IndexManagementLocator = LocatorPublic; diff --git a/x-pack/plugins/index_management/public/application/services/routing.ts b/x-pack/plugins/index_management/public/application/services/routing.ts index bce7a14f03e46..89143bbd79d02 100644 --- a/x-pack/plugins/index_management/public/application/services/routing.ts +++ b/x-pack/plugins/index_management/public/application/services/routing.ts @@ -11,6 +11,7 @@ import { Section } from '../../../common/constants'; import type { IndexDetailsTabId } from '../../../common/constants'; import { ExtensionsService } from '../../services/extensions_service'; import { IndexDetailsSection } from '../../../common/constants'; + export const getTemplateListLink = () => `/templates`; export const getTemplateDetailsLink = (name: string, isLegacy?: boolean) => { @@ -81,6 +82,11 @@ export const getComponentTemplatesLink = (usedByTemplateName?: string) => { } return url; }; + +export const getComponentTemplateDetailLink = (name: string) => { + return `/component_templates/${encodeURIComponent(name)}`; +}; + export const navigateToIndexDetailsPage = ( indexName: string, indicesListURLParams: string, diff --git a/x-pack/plugins/index_management/public/locator.test.ts b/x-pack/plugins/index_management/public/locator.test.ts index 712223d7cbfe4..49b9890259d9c 100644 --- a/x-pack/plugins/index_management/public/locator.test.ts +++ b/x-pack/plugins/index_management/public/locator.test.ts @@ -34,4 +34,26 @@ describe('Index Management URL locator', () => { }); expect(path).toBe('/data/index_management/data_streams/test'); }); + + test('locator returns the correct url for index_template', async () => { + const indexTemplateName = 'test@custom'; + const { path } = await locator.getLocation({ + page: 'index_template', + indexTemplate: indexTemplateName, + }); + expect(path).toBe( + encodeURI(`/data/index_management/templates/${encodeURIComponent(indexTemplateName)}`) + ); + }); + + test('locator returns the correct url for component_template', async () => { + const componentTemplateName = 'log@custom'; + const { path } = await locator.getLocation({ + page: 'component_template', + componentTemplate: componentTemplateName, + }); + expect(path).toBe( + `/data/index_management/component_templates/${encodeURIComponent(componentTemplateName)}` + ); + }); }); diff --git a/x-pack/plugins/index_management/public/locator.ts b/x-pack/plugins/index_management/public/locator.ts index d32d33573507d..c5411aded71a4 100644 --- a/x-pack/plugins/index_management/public/locator.ts +++ b/x-pack/plugins/index_management/public/locator.ts @@ -8,7 +8,11 @@ import { ManagementAppLocator } from '@kbn/management-plugin/common'; import { LocatorDefinition } from '@kbn/share-plugin/public'; import { IndexManagementLocatorParams } from '@kbn/index-management-shared-types'; -import { getDataStreamDetailsLink } from './application/services/routing'; +import { + getComponentTemplateDetailLink, + getDataStreamDetailsLink, + getTemplateDetailsLink, +} from './application/services/routing'; import { PLUGIN } from '../common/constants'; export const INDEX_MANAGEMENT_LOCATOR_ID = 'INDEX_MANAGEMENT_LOCATOR_ID'; @@ -37,6 +41,18 @@ export class IndexManagementLocatorDefinition path: location.path + getDataStreamDetailsLink(params.dataStreamName!), }; } + case 'index_template': { + return { + ...location, + path: location.path + getTemplateDetailsLink(params.indexTemplate), + }; + } + case 'component_template': { + return { + ...location, + path: location.path + getComponentTemplateDetailLink(params.componentTemplate), + }; + } } }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index bfbb2bc1cd5d1..903d7f0607663 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -134,22 +134,39 @@ export const degradedFieldAnalysisRt = rt.intersection([ type: rt.string, ignore_above: rt.number, }), + defaultPipeline: rt.string, }), ]); export type DegradedFieldAnalysis = rt.TypeOf; -export const dataStreamSettingsRt = rt.intersection([ +export const updateFieldLimitResponseRt = rt.intersection([ rt.type({ - lastBackingIndexName: rt.string, + isComponentTemplateUpdated: rt.union([rt.boolean, rt.undefined]), + isLatestBackingIndexUpdated: rt.union([rt.boolean, rt.undefined]), + customComponentTemplateName: rt.string, }), rt.partial({ - createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless - integration: rt.string, - datasetUserPrivileges: datasetUserPrivilegesRt, + error: rt.string, }), ]); +export type UpdateFieldLimitResponse = rt.TypeOf; + +export const dataStreamRolloverResponseRt = rt.type({ + acknowledged: rt.boolean, +}); + +export type DataStreamRolloverResponse = rt.TypeOf; + +export const dataStreamSettingsRt = rt.partial({ + lastBackingIndexName: rt.string, + indexTemplate: rt.string, + createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless + integration: rt.string, + datasetUserPrivileges: datasetUserPrivilegesRt, +}); + export type DataStreamSettings = rt.TypeOf; export const dataStreamDetailsRt = rt.partial({ diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts index 66b7567a2b60c..ce74552b581b9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts @@ -14,3 +14,8 @@ export interface AnalyzeDegradedFieldsParams { lastBackingIndex: string; degradedField: string; } + +export interface UpdateFieldLimitParams { + dataStream: string; + newFieldLimit: number; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index e5b660b31de10..1026dd8ea58d3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -500,3 +500,182 @@ export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate 'This issue was detected in an older version of the dataset, but not in the most recent version.', } ); + +export const possibleMitigationTitle = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigationTitle', + { + defaultMessage: 'Possible mitigation', + } +); + +export const increaseFieldMappingLimitTitle = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.increaseFieldMappingLimitTitle', + { + defaultMessage: 'Increase field mapping limit', + } +); + +export const fieldLimitMitigationDescriptionText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationDescription', + { + defaultMessage: + 'The field mapping limit sets the maximum number of fields in an index. When exceeded, additional fields are ignored. To prevent this, increase your field mapping limit.', + } +); + +export const fieldLimitMitigationConsiderationText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations', + { + defaultMessage: 'Before changing the field limit, consider the following:', + } +); + +export const fieldLimitMitigationConsiderationText1 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations1', + { + defaultMessage: 'Increasing the field limit could slow cluster performance.', + } +); + +export const fieldLimitMitigationConsiderationText2 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations2', + { + defaultMessage: 'Increasing the field limit also resolves field limit issues for other fields.', + } +); + +export const fieldLimitMitigationConsiderationText3 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations3', + { + defaultMessage: + 'This change applies to the [name] component template and affects all namespaces in the template.', + } +); + +export const fieldLimitMitigationConsiderationText4 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations4', + { + defaultMessage: + 'You need to roll over affected data streams to apply mapping changes to component templates.', + } +); + +export const fieldLimitMitigationCurrentLimitLabelText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationCurrentLimitLabelText', + { + defaultMessage: 'Current limit', + } +); + +export const fieldLimitMitigationNewLimitButtonText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitButtonText', + { + defaultMessage: 'New limit', + } +); + +export const fieldLimitMitigationNewLimitPlaceholderText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitPlaceholderText', + { + defaultMessage: 'New field limit', + } +); + +export const fieldLimitMitigationApplyButtonText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationApplyButtonText', + { + defaultMessage: 'Apply', + } +); + +export const otherMitigationsLoadingAriaText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsLoadingText', + { + defaultMessage: 'Loading possible mitigations', + } +); + +export const otherMitigationsCustomComponentTemplate = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomComponentTemplate', + { + defaultMessage: 'Add or edit custom component template', + } +); + +export const otherMitigationsCustomIngestPipeline = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomIngestPipeline', + { + defaultMessage: 'Add or edit custom ingest pipeline', + } +); + +export const fieldLimitMitigationOfficialDocumentation = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationOfficialDocumentation', + { + defaultMessage: 'Documentation', + } +); + +export const fieldLimitMitigationSuccessMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessMessage', + { + defaultMessage: 'New limit set!', + } +); + +export const fieldLimitMitigationSuccessComponentTemplateLinkText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessComponentTemplateLinkText', + { + defaultMessage: 'See component template', + } +); + +export const fieldLimitMitigationPartiallyFailedMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessage', + { + defaultMessage: 'Changes not applied to new data', + } +); + +export const fieldLimitMitigationFailedMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessage', + { + defaultMessage: 'Changes not applied', + } +); + +export const fieldLimitMitigationFailedMessageDescription = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessageDescription', + { + defaultMessage: 'Failed to set new limit', + } +); + +export const fieldLimitMitigationPartiallyFailedMessageDescription = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessageDescription', + { + defaultMessage: + 'The component template was successfully updated with the new field limit, but the changes were not applied to the most recent backing index. Perform a rollover to apply your changes to new data.', + } +); + +export const fieldLimitMitigationRolloverButton = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationRolloverButton', + { + defaultMessage: 'Rollover', + } +); + +export const manualMitigationCustomPipelineCopyPipelineNameAriaText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.copyPipelineNameAriaText', + { + defaultMessage: 'Copy pipeline name', + } +); + +export const manualMitigationCustomPipelineCreateEditPipelineLink = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.createEditPipelineLink', + { + defaultMessage: 'create or edit the pipeline', + } +); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts new file mode 100644 index 0000000000000..0be7b84137c83 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * There are index templates like this metrics-apm.service_transaction.10m@template which exists. + * Hence this @ needs to be removed to derive the custom component template name. + */ +export function getComponentTemplatePrefixFromIndexTemplate(indexTemplate: string) { + if (indexTemplate.includes('@')) { + return indexTemplate.split('@')[0]; + } + + return indexTemplate; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx index 1e6bda781d733..3bcee0bbc89b3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx @@ -38,7 +38,7 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) degradedFieldValues, isDegradedFieldsLoading, isAnalysisInProgress, - degradedFieldAnalysisResult, + degradedFieldAnalysisFormattedResult, degradedFieldAnalysis, } = useDegradedFields(); @@ -94,9 +94,12 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) grow={2} >
- + - {degradedFieldAnalysisResult?.potentialCause} + {degradedFieldAnalysisFormattedResult?.potentialCause}
@@ -125,52 +128,53 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) )} - {!isAnalysisInProgress && degradedFieldAnalysisResult?.shouldDisplayValues && ( - <> - - - - {degradedFieldMaximumCharacterLimitColumnName} - - - + - {degradedFieldAnalysis?.fieldMapping?.ignore_above} - - - - - - - {degradedFieldValuesColumnName} - - - + + {degradedFieldMaximumCharacterLimitColumnName} + + + + {degradedFieldAnalysis?.fieldMapping?.ignore_above} + + + + - - {degradedFieldValues?.values.map((value, idx) => ( - - - {value} - - - ))} - - - - - - )} + + + {degradedFieldValuesColumnName} + + + + + {degradedFieldValues?.values.map((value, idx) => ( + + + {value} + + + ))} + + + + + + )} ); }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx index 189b3ceefe37c..bb72b4f6de20f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx @@ -6,6 +6,7 @@ */ import React, { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiBadge, EuiFlyout, @@ -20,6 +21,7 @@ import { EuiButtonIcon, EuiToolTip, } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; import { NavigationSource } from '../../../services/telemetry'; import { useDatasetDetailsRedirectLinkTelemetry, @@ -38,11 +40,18 @@ import { } from '../../../../common/translations'; import { DegradedFieldInfo } from './field_info'; import { _IGNORED } from '../../../../common/es_fields'; +import { PossibleMitigations } from './possible_mitigations'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DegradedFieldFlyout() { - const { closeDegradedFieldFlyout, expandedDegradedField, renderedItems } = useDegradedFields(); + const { + closeDegradedFieldFlyout, + expandedDegradedField, + renderedItems, + isAnalysisInProgress, + degradedFieldAnalysisFormattedResult, + } = useDegradedFields(); const { dataStreamSettings, datasetDetails, timeRange } = useDatasetQualityDetailsState(); const pushedFlyoutTitleId = useGeneratedHtmlId({ prefix: 'pushedFlyoutTitle', @@ -118,9 +127,42 @@ export default function DegradedFieldFlyout() { )} + {isUserViewingTheIssueOnLatestBackingIndex && + !isAnalysisInProgress && + degradedFieldAnalysisFormattedResult && + !degradedFieldAnalysisFormattedResult.identifiedUsingHeuristics && ( + <> + + + + {i18n.translate( + 'xpack.datasetQuality.degradedFieldFlyout.strong.fieldLimitLabel', + { defaultMessage: 'field limit' } + )} + + ), + }} + /> + + + )} + {isUserViewingTheIssueOnLatestBackingIndex && ( + <> + + + + )} ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx new file mode 100644 index 0000000000000..0dd80bb120e54 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiLink } from '@elastic/eui'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../common/translations'; + +export function FieldLimitDocLink() { + const { + services: { docLinks }, + } = useKibanaContextForPlugin(); + + return ( + + {fieldLimitMitigationOfficialDocumentation} + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx new file mode 100644 index 0000000000000..1056713ac2070 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { + EuiAccordion, + EuiHorizontalRule, + EuiPanel, + EuiSpacer, + EuiText, + EuiTitle, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { + fieldLimitMitigationConsiderationText, + fieldLimitMitigationConsiderationText1, + fieldLimitMitigationConsiderationText2, + fieldLimitMitigationConsiderationText3, + fieldLimitMitigationConsiderationText4, + fieldLimitMitigationDescriptionText, + increaseFieldMappingLimitTitle, +} from '../../../../../../common/translations'; +import { useDegradedFields } from '../../../../../hooks'; +import { IncreaseFieldMappingLimit } from './increase_field_mapping_limit'; +import { FieldLimitDocLink } from './field_limit_documentation_link'; +import { MessageCallout } from './message_callout'; + +export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean }) { + const accordionId = useGeneratedHtmlId({ + prefix: increaseFieldMappingLimitTitle, + }); + + const { degradedFieldAnalysis } = useDegradedFields(); + + const accordionTitle = ( + +
{increaseFieldMappingLimitTitle}
+
+ ); + + return ( + + + + {fieldLimitMitigationDescriptionText} + + + +

{fieldLimitMitigationConsiderationText}

+ +
    +
  • {fieldLimitMitigationConsiderationText1}
  • +
  • {fieldLimitMitigationConsiderationText2}
  • +
  • {fieldLimitMitigationConsiderationText3}
  • +
  • {fieldLimitMitigationConsiderationText4}
  • +
+
+ + {isIntegration && ( + <> + + + + + + )} + +
+
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx new file mode 100644 index 0000000000000..158a5e5eba460 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiFieldText, + EuiFormRow, + EuiButton, + EuiFieldNumber, +} from '@elastic/eui'; +import { + fieldLimitMitigationApplyButtonText, + fieldLimitMitigationCurrentLimitLabelText, + fieldLimitMitigationNewLimitButtonText, + fieldLimitMitigationNewLimitPlaceholderText, +} from '../../../../../../common/translations'; +import { useDegradedFields } from '../../../../../hooks'; + +export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit: number }) { + // Propose the user a 30% increase over the current limit + const proposedNewLimit = Math.round(totalFieldLimit * 1.3); + const [newFieldLimit, setNewFieldLimit] = useState(proposedNewLimit); + const [isInvalid, setIsInvalid] = useState(false); + const { updateNewFieldLimit, isMitigationInProgress } = useDegradedFields(); + + const validateNewLimit = (newLimit: string) => { + const parsedLimit = parseInt(newLimit, 10); + setNewFieldLimit(parsedLimit); + if (totalFieldLimit > parsedLimit) { + setIsInvalid(true); + } else { + setIsInvalid(false); + } + }; + + return ( + + + + + + + + + validateNewLimit(e.target.value)} + aria-label={fieldLimitMitigationNewLimitPlaceholderText} + isInvalid={isInvalid} + min={totalFieldLimit + 1} + /> + + + + + updateNewFieldLimit(newFieldLimit)} + isLoading={isMitigationInProgress} + > + {fieldLimitMitigationApplyButtonText} + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx new file mode 100644 index 0000000000000..168ae4df575e9 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiButton, EuiCallOut, EuiLink } from '@elastic/eui'; +import { + fieldLimitMitigationFailedMessage, + fieldLimitMitigationFailedMessageDescription, + fieldLimitMitigationPartiallyFailedMessage, + fieldLimitMitigationPartiallyFailedMessageDescription, + fieldLimitMitigationRolloverButton, + fieldLimitMitigationSuccessComponentTemplateLinkText, + fieldLimitMitigationSuccessMessage, +} from '../../../../../../common/translations'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; +import { useKibanaContextForPlugin } from '../../../../../utils'; + +export function MessageCallout() { + const { + isMitigationInProgress, + newFieldLimitData, + isRolloverRequired, + isMitigationAppliedSuccessfully, + } = useDegradedFields(); + const { error: serverError } = newFieldLimitData ?? {}; + + if (serverError) { + return ; + } + + if (!isMitigationInProgress && isRolloverRequired) { + return ; + } + + if (!isMitigationInProgress && isMitigationAppliedSuccessfully) { + return ; + } + + return null; +} + +export function SuccessCallout() { + const { + services: { + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); + const { name } = datasetDetails; + + const componentTemplateUrl = locators.get('INDEX_MANAGEMENT_LOCATOR_ID')?.useUrl({ + page: 'component_template', + componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom`, + }); + + return ( + + + {fieldLimitMitigationSuccessComponentTemplateLinkText} + + + ); +} + +export function ManualRolloverCallout() { + const { triggerRollover, isRolloverInProgress } = useDegradedFields(); + return ( + +

{fieldLimitMitigationPartiallyFailedMessageDescription}

+ + {fieldLimitMitigationRolloverButton} + +
+ ); +} + +export function ErrorCallout() { + return ( + +

{fieldLimitMitigationFailedMessageDescription}

+
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx new file mode 100644 index 0000000000000..34f39f25a67ec --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSpacer } from '@elastic/eui'; +import { ManualMitigations } from './manual'; +import { FieldMappingLimit } from './field_limit/field_mapping_limit'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../hooks'; +import { PossibleMitigationTitle } from './title'; + +export function PossibleMitigations() { + const { degradedFieldAnalysis, isAnalysisInProgress } = useDegradedFields(); + const { integrationDetails } = useDatasetQualityDetailsState(); + const isIntegration = Boolean(integrationDetails?.integration); + + return ( + !isAnalysisInProgress && ( +
+ + + {degradedFieldAnalysis?.isFieldLimitIssue && ( + <> + + + + )} + +
+ ) + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx new file mode 100644 index 0000000000000..54bbe91f2f2e1 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants'; +import { EuiFlexGroup, EuiIcon, EuiLink, EuiPanel, EuiTitle } from '@elastic/eui'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; +import { otherMitigationsCustomComponentTemplate } from '../../../../../../common/translations'; + +export function CreateEditComponentTemplateLink({ isIntegration }: { isIntegration: boolean }) { + const { + services: { + application, + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + + const [indexTemplatePath, setIndexTemplatePath] = useState(null); + const [componentTemplatePath, setComponentTemplatePath] = useState(null); + + const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); + const { name } = datasetDetails; + + const indexManagementLocator = locators.get('INDEX_MANAGEMENT_LOCATOR_ID'); + + useEffect(() => { + indexManagementLocator + ?.getLocation({ + page: 'index_template', + indexTemplate: dataStreamSettings?.indexTemplate ?? '', + }) + .then(({ path }) => setIndexTemplatePath(path)); + indexManagementLocator + ?.getLocation({ + page: 'component_template', + componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom`, + }) + .then(({ path }) => setComponentTemplatePath(path)); + }, [ + locators, + setIndexTemplatePath, + dataStreamSettings?.indexTemplate, + indexManagementLocator, + name, + ]); + + const templateUrl = isIntegration ? componentTemplatePath : indexTemplatePath; + + const onClickHandler = useCallback(async () => { + const options = { + openInNewTab: true, + ...(templateUrl && { path: templateUrl }), + }; + + await application.navigateToApp(MANAGEMENT_APP_ID, options); + }, [application, templateUrl]); + + return ( + + + + + +

{otherMitigationsCustomComponentTemplate}

+
+
+
+
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx new file mode 100644 index 0000000000000..f931f3461fb57 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSkeletonRectangle, EuiSpacer } from '@elastic/eui'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; +import { CreateEditComponentTemplateLink } from './component_template_link'; +import { CreateEditPipelineLink } from './pipeline_link'; +import { otherMitigationsLoadingAriaText } from '../../../../../../common/translations'; + +export function ManualMitigations() { + const { integrationDetails, loadingState, dataStreamSettings } = useDatasetQualityDetailsState(); + const isIntegrationPresentInSettings = dataStreamSettings?.integration; + const isIntegration = !!integrationDetails?.integration; + const { dataStreamSettingsLoading, integrationDetailsLoadings } = loadingState; + + const hasIntegrationCheckCompleted = + !dataStreamSettingsLoading && + ((isIntegrationPresentInSettings && !integrationDetailsLoadings) || + !isIntegrationPresentInSettings); + + return ( + + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx new file mode 100644 index 0000000000000..6179a3ed0736c --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useMemo } from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; +import { + copyToClipboard, + EuiAccordion, + EuiButtonIcon, + EuiFieldText, + EuiHorizontalRule, + EuiLink, + EuiPanel, + EuiSpacer, + EuiTitle, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { + manualMitigationCustomPipelineCopyPipelineNameAriaText, + manualMitigationCustomPipelineCreateEditPipelineLink, + otherMitigationsCustomIngestPipeline, +} from '../../../../../../common/translations'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; + +const AccordionTitle = () => ( + +
{otherMitigationsCustomIngestPipeline}
+
+); + +export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boolean }) { + const { + services: { + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + + const accordionId = useGeneratedHtmlId({ + prefix: otherMitigationsCustomIngestPipeline, + }); + + const { datasetDetails } = useDatasetQualityDetailsState(); + const { type, name } = datasetDetails; + + const pipelineName = useMemo( + () => (isIntegration ? `${type}-${name}@custom` : `${type}@custom`), + [isIntegration, type, name] + ); + + const ingestPipelineLocator = locators.get('INGEST_PIPELINES_APP_LOCATOR'); + + const pipelineUrl = ingestPipelineLocator?.useUrl( + { pipelineId: pipelineName, page: 'pipelines_list' }, + {}, + [pipelineName] + ); + + const onClickHandler = useCallback(() => { + copyToClipboard(pipelineName); + }, [pipelineName]); + + return ( + + } + paddingSize="none" + initialIsOpen={true} + data-test-subj="datasetQualityManualMitigationsPipelineAccordion" + > + + + {i18n.translate('xpack.datasetQuality.editPipeline.strong.Label', { + defaultMessage: '1.', + })} + + ), + }} + /> + + + } + readOnly={true} + aria-label={manualMitigationCustomPipelineCopyPipelineNameAriaText} + value={pipelineName} + data-test-subj="datasetQualityManualMitigationsPipelineName" + fullWidth + /> + + + {i18n.translate('xpack.datasetQuality.editPipeline.strong.Label', { + defaultMessage: '2.', + })} + + ), + createEditPipelineLink: ( + + {manualMitigationCustomPipelineCreateEditPipelineLink} + + ), + }} + /> + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx new file mode 100644 index 0000000000000..93e253b0f849c --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiBetaBadge, EuiFlexGroup, EuiIcon, EuiTitle } from '@elastic/eui'; + +import { + overviewQualityIssuesAccordionTechPreviewBadge, + possibleMitigationTitle, +} from '../../../../../common/translations'; + +export function PossibleMitigationTitle() { + return ( + + + +

{possibleMitigationTitle}

+
+ +
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx index b33bd11dbe3a6..0cdc460cd56dc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx @@ -45,6 +45,7 @@ export function DegradedFields() { aria-describedby={toggleTextSwitchId} compressed data-test-subj="datasetQualityDetailsOverviewDegradedFieldToggleSwitch" + css={{ marginRight: '5px' }} /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts index 78ad0e53dd5e2..49ceb50abc3cd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts @@ -104,20 +104,25 @@ export function useDegradedFields() { }, [service]); const degradedFieldValues = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.ignoredValues.done') + state.matches('initializing.degradedFieldFlyout.open.initialized.ignoredValues.done') ? state.context.degradedFieldValues : undefined ); const degradedFieldAnalysis = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.analyze.done') + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating') || + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + ) || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error') ? state.context.degradedFieldAnalysis : undefined ); - // This piece only cater field limit issue at the moment. - // In future this will cater the other 2 reasons as well - const degradedFieldAnalysisResult = useMemo(() => { + const degradedFieldAnalysisFormattedResult = useMemo(() => { if (!degradedFieldAnalysis) { return undefined; } @@ -127,8 +132,8 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldLimitExceeded, tooltipContent: degradedFieldCauseFieldLimitExceededTooltip, - shouldDisplayMitigation: true, - shouldDisplayValues: false, + shouldDisplayIgnoredValuesAndLimit: false, + identifiedUsingHeuristics: true, }; } @@ -143,8 +148,8 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldIgnored, tooltipContent: degradedFieldCauseFieldIgnoredTooltip, - shouldDisplayMitigation: false, - shouldDisplayValues: true, + shouldDisplayIgnoredValuesAndLimit: true, + identifiedUsingHeuristics: true, }; } } @@ -153,19 +158,59 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldMalformed, tooltipContent: degradedFieldCauseFieldMalformedTooltip, - shouldDisplayMitigation: false, - shouldDisplayValues: false, + shouldDisplayIgnoredValuesAndLimit: false, + identifiedUsingHeuristics: false, // TODO: Add heuristics to identify ignore_malformed issues }; }, [degradedFieldAnalysis, degradedFieldValues]); const isDegradedFieldsValueLoading = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.ignoredValues.fetching'); + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' + ); + }); + + const isRolloverRequired = useSelector(service, (state) => { + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + ); + }); + + const isMitigationAppliedSuccessfully = useSelector(service, (state) => { + return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success'); }); const isAnalysisInProgress = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.analyze.fetching'); + return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing'); + }); + + const isRolloverInProgress = useSelector(service, (state) => { + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver' + ); + }); + + const updateNewFieldLimit = useCallback( + (newFieldLimit: number) => { + service.send({ type: 'SET_NEW_FIELD_LIMIT', newFieldLimit }); + }, + [service] + ); + + const isMitigationInProgress = useSelector(service, (state) => { + return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating'); }); + const newFieldLimitData = useSelector(service, (state) => + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error') + ? state.context.fieldLimit + : undefined + ); + + const triggerRollover = useCallback(() => { + service.send('ROLLOVER_DATA_STREAM'); + }, [service]); + return { isDegradedFieldsLoading, pagination, @@ -181,9 +226,16 @@ export function useDegradedFields() { isDegradedFieldsValueLoading, isAnalysisInProgress, degradedFieldAnalysis, - degradedFieldAnalysisResult, + degradedFieldAnalysisFormattedResult, toggleCurrentQualityIssues, showCurrentQualityIssues, expandedRenderedItem, + updateNewFieldLimit, + isMitigationInProgress, + isRolloverInProgress, + newFieldLimitData, + isRolloverRequired, + isMitigationAppliedSuccessfully, + triggerRollover, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 9175d06e105b4..827cd4b0a1e49 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,6 +8,8 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { + DataStreamRolloverResponse, + dataStreamRolloverResponseRt, DegradedFieldAnalysis, degradedFieldAnalysisRt, DegradedFieldValues, @@ -19,6 +21,8 @@ import { IntegrationDashboardsResponse, integrationDashboardsRT, IntegrationResponse, + UpdateFieldLimitResponse, + updateFieldLimitResponseRt, } from '../../../common/api_types'; import { DataStreamDetails, @@ -37,6 +41,7 @@ import { Integration } from '../../../common/data_streams_stats/integration'; import { AnalyzeDegradedFieldsParams, GetDataStreamIntegrationParams, + UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; import { DatasetQualityError } from '../../../common/errors'; @@ -196,4 +201,46 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { new DatasetQualityError(`Failed to decode the analysis response: ${message}`) )(response); } + + public async setNewFieldLimit({ + dataStream, + newFieldLimit, + }: UpdateFieldLimitParams): Promise { + const response = await this.http + .put( + `/internal/dataset_quality/data_streams/${dataStream}/update_field_limit`, + { body: JSON.stringify({ newFieldLimit }) } + ) + .catch((error) => { + throw new DatasetQualityError(`Failed to set new Limit: ${error.message}`, error); + }); + + const decodedResponse = decodeOrThrow( + updateFieldLimitResponseRt, + (message: string) => + new DatasetQualityError(`Failed to decode setting of new limit response: ${message}"`) + )(response); + + return decodedResponse; + } + + public async rolloverDataStream({ + dataStream, + }: { + dataStream: string; + }): Promise { + const response = await this.http + .post( + `/internal/dataset_quality/data_streams/${dataStream}/rollover` + ) + .catch((error) => { + throw new DatasetQualityError(`Failed to rollover datastream": ${error}`, error); + }); + + return decodeOrThrow( + dataStreamRolloverResponseRt, + (message: string) => + new DatasetQualityError(`Failed to decode rollover response: ${message}"`) + )(response); + } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index a2f7db99e5af1..6eac8bd732840 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -20,8 +20,15 @@ import { import { AnalyzeDegradedFieldsParams, GetDataStreamIntegrationParams, + UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; -import { Dashboard, DegradedFieldAnalysis, DegradedFieldValues } from '../../../common/api_types'; +import { + Dashboard, + DataStreamRolloverResponse, + DegradedFieldAnalysis, + DegradedFieldValues, + UpdateFieldLimitResponse, +} from '../../../common/api_types'; export type DataStreamDetailsServiceSetup = void; @@ -47,4 +54,6 @@ export interface IDataStreamDetailsClient { params: GetDataStreamIntegrationParams ): Promise; analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise; + setNewFieldLimit(params: UpdateFieldLimitParams): Promise; + rolloverDataStream(params: { dataStream: string }): Promise; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts index b501fd02bdcf3..f5fdd063492a3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts @@ -59,3 +59,28 @@ export const fetchDataStreamIntegrationFailedNotifier = ( text: error.message, }); }; + +export const updateFieldLimitFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.details.updateFieldLimitFailed', { + defaultMessage: "We couldn't update the field limit.", + }), + text: error.message, + }); +}; + +export const rolloverDataStreamFailedNotifier = ( + toasts: IToasts, + error: Error, + dataStream: string +) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.details.rolloverDataStreamFailed', { + defaultMessage: "We couldn't rollover the data stream: {dataStream}.", + values: { + dataStream, + }, + }), + text: error.message, + }); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 352aff140c275..8ac65a7dca4a7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -25,6 +25,7 @@ import { DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, + UpdateFieldLimitResponse, } from '../../../common/api_types'; import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications'; import { @@ -33,6 +34,8 @@ import { fetchDataStreamSettingsFailedNotifier, fetchDataStreamIntegrationFailedNotifier, fetchIntegrationDashboardsFailedNotifier, + updateFieldLimitFailedNotifier, + rolloverDataStreamFailedNotifier, } from './notifications'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -189,10 +192,6 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, done: { on: { - UPDATE_TIME_RANGE: { - target: 'fetching', - actions: ['resetDegradedFieldPageAndRowsPerPage'], - }, UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { target: 'done', actions: ['storeDegradedFieldTableOptions'], @@ -200,7 +199,10 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( OPEN_DEGRADED_FIELD_FLYOUT: { target: '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', - actions: ['storeExpandedDegradedField'], + actions: [ + 'storeExpandedDegradedField', + 'resetFieldLimitServerResponse', + ], }, TOGGLE_CURRENT_QUALITY_ISSUES: { target: 'fetching', @@ -282,48 +284,105 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, open: { - type: 'parallel', + initial: 'initialized', states: { - ignoredValues: { - initial: 'fetching', + initialized: { + type: 'parallel', states: { - fetching: { - invoke: { - src: 'loadDegradedFieldValues', - onDone: { - target: 'done', - actions: ['storeDegradedFieldValues'], - }, - onError: [ - { - target: '#DatasetQualityDetailsController.indexNotFound', - cond: 'isIndexNotFoundError', - }, - { - target: 'done', + ignoredValues: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDegradedFieldValues', + onDone: { + target: 'done', + actions: ['storeDegradedFieldValues'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], }, - ], + }, + done: {}, }, }, - done: {}, - }, - }, - analyze: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'analyzeDegradedField', - onDone: { - target: 'done', - actions: ['storeDegradedFieldAnalysis'], + mitigation: { + initial: 'analyzing', + states: { + analyzing: { + invoke: { + src: 'analyzeDegradedField', + onDone: { + target: 'analyzed', + actions: ['storeDegradedFieldAnalysis'], + }, + onError: { + target: 'analyzed', + }, + }, }, - onError: { - target: 'done', + analyzed: { + on: { + SET_NEW_FIELD_LIMIT: { + target: 'mitigating', + actions: 'storeNewFieldLimit', + }, + }, + }, + mitigating: { + invoke: { + src: 'saveNewFieldLimit', + onDone: [ + { + target: 'askingForRollover', + actions: 'storeNewFieldLimitResponse', + cond: 'hasFailedToUpdateLastBackingIndex', + }, + { + target: 'success', + actions: 'storeNewFieldLimitResponse', + }, + ], + onError: { + target: 'error', + actions: [ + 'storeNewFieldLimitErrorResponse', + 'notifySaveNewFieldLimitError', + ], + }, + }, + }, + askingForRollover: { + on: { + ROLLOVER_DATA_STREAM: { + target: 'rollingOver', + }, + }, + }, + rollingOver: { + invoke: { + src: 'rolloverDataStream', + onDone: { + target: 'success', + actions: ['raiseForceTimeRangeRefresh'], + }, + onError: { + target: 'error', + actions: 'notifySaveNewFieldLimitError', + }, + }, }, + success: {}, + error: {}, }, }, - done: {}, }, }, }, @@ -482,9 +541,28 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( isIndexNotFoundError: true, }; }), + storeNewFieldLimit: assign((_, event) => { + return 'newFieldLimit' in event + ? { fieldLimit: { newFieldLimit: event.newFieldLimit } } + : {}; + }), + storeNewFieldLimitResponse: assign( + (context, event: DoneInvokeEvent) => { + return 'data' in event + ? { fieldLimit: { ...context.fieldLimit, result: event.data, error: false } } + : {}; + } + ), + storeNewFieldLimitErrorResponse: assign((context) => { + return { fieldLimit: { ...context.fieldLimit, error: true } }; + }), + resetFieldLimitServerResponse: assign(() => ({ + fieldLimit: undefined, + })), + raiseForceTimeRangeRefresh: raise('UPDATE_TIME_RANGE'), }, guards: { - checkIfActionForbidden: (context, event) => { + checkIfActionForbidden: (_, event) => { return ( 'data' in event && typeof event.data === 'object' && @@ -516,6 +594,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( hasNoDegradedFieldsSelected: (context) => { return !Boolean(context.expandedDegradedField); }, + hasFailedToUpdateLastBackingIndex: (_, event) => { + return ( + 'data' in event && + typeof event.data === 'object' && + 'isLatestBackingIndexUpdated' in event.data && + !event.data.isLatestBackingIndexUpdated + ); + }, }, } ); @@ -552,6 +638,10 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ 'dataStreamSettings' in context ? context.dataStreamSettings?.integration : undefined; return fetchDataStreamIntegrationFailedNotifier(toasts, event.data, integrationName); }, + notifySaveNewFieldLimitError: (_context, event: DoneInvokeEvent) => + updateFieldLimitFailedNotifier(toasts, event.data), + notifyRolloverDataStreamError: (context, event: DoneInvokeEvent) => + rolloverDataStreamFailedNotifier(toasts, event.data, context.dataStream), }, services: { checkDatasetIsAggregatable: (context) => { @@ -603,7 +693,8 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ dataStream: context.showCurrentQualityIssues && 'dataStreamSettings' in context && - context.dataStreamSettings + context.dataStreamSettings && + context.dataStreamSettings.lastBackingIndexName ? context.dataStreamSettings.lastBackingIndexName : context.dataStream, start, @@ -661,6 +752,21 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ return Promise.resolve(); }, + saveNewFieldLimit: (context) => { + if ('fieldLimit' in context && context.fieldLimit && context.fieldLimit.newFieldLimit) { + return dataStreamDetailsClient.setNewFieldLimit({ + dataStream: context.dataStream, + newFieldLimit: context.fieldLimit.newFieldLimit, + }); + } + + return Promise.resolve(); + }, + rolloverDataStream: (context) => { + return dataStreamDetailsClient.rolloverDataStream({ + dataStream: context.dataStream, + }); + }, }, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index cdf3bfa579e55..cdebcfbe53d86 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -10,12 +10,14 @@ import type { DegradedFieldSortField } from '../../hooks'; import { Dashboard, DataStreamDetails, + DataStreamRolloverResponse, DataStreamSettings, DegradedField, DegradedFieldAnalysis, DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, + UpdateFieldLimitResponse, } from '../../../common/api_types'; import { TableCriteria, TimeRangeConfig } from '../../../common/types'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -37,6 +39,12 @@ export interface DegradedFieldsWithData { data: DegradedField[]; } +export interface FieldLimit { + newFieldLimit?: number; + result?: UpdateFieldLimitResponse; + error?: boolean; +} + export interface WithDefaultControllerState { dataStream: string; degradedFields: DegradedFieldsTableConfig; @@ -48,6 +56,7 @@ export interface WithDefaultControllerState { integration?: Integration; expandedDegradedField?: string; isNonAggregatable?: boolean; + fieldLimit?: FieldLimit; } export interface WithDataStreamDetails { @@ -87,6 +96,16 @@ export interface WithDegradeFieldAnalysis { degradedFieldAnalysis: DegradedFieldAnalysis; } +export interface WithNewFieldLimit { + fieldLimit?: FieldLimit & { + newFieldLimit: number; + }; +} + +export interface WithNewFieldLimitResponse { + fieldLimit: FieldLimit; +} + export type DefaultDatasetQualityDetailsContext = Pick< WithDefaultControllerState, 'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues' @@ -128,38 +147,50 @@ export type DatasetQualityDetailsControllerTypeState = } | { value: - | 'initializing.degradedFieldFlyout.open.ignoredValues.fetching' - | 'initializing.degradedFieldFlyout.open.analyze.fetching'; + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'; + context: WithDefaultControllerState & WithDataStreamSettings; + } + | { + value: + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'; + context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration; + } + | { + value: 'initializing.degradedFieldFlyout.open'; + context: WithDefaultControllerState; + } + | { + value: + | 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing'; context: WithDefaultControllerState & WithDegradedFieldsData; } | { - value: 'initializing.degradedFieldFlyout.open.ignoredValues.done'; + value: 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.done'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues; } | { - value: 'initializing.degradedFieldFlyout.open.analyze.done'; + value: + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.success' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.error'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradeFieldAnalysis; } | { - value: 'initializing.degradedFieldFlyout.open'; + value: 'initializing.degradedFieldFlyout.open.initialized.mitigation.success'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues & - WithDegradeFieldAnalysis; - } - | { - value: - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'; - context: WithDefaultControllerState & WithDataStreamSettings; - } - | { - value: - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'; - context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration; + WithDegradeFieldAnalysis & + WithNewFieldLimit & + WithNewFieldLimitResponse; }; export type DatasetQualityDetailsControllerContext = @@ -188,6 +219,13 @@ export type DatasetQualityDetailsControllerEvent = type: 'UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA'; degraded_field_criteria: TableCriteria; } + | { + type: 'SET_NEW_FIELD_LIMIT'; + newFieldLimit: number; + } + | { + type: 'ROLLOVER_DATA_STREAM'; + } | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent @@ -197,4 +235,6 @@ export type DatasetQualityDetailsControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent - | DoneInvokeEvent; + | DoneInvokeEvent + | DoneInvokeEvent + | DoneInvokeEvent; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index eb1d70b867dc4..288eff11b92a8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -29,8 +29,6 @@ export async function getDataStreamSettings({ esClient: ElasticsearchClient; dataStream: string; }): Promise { - throwIfInvalidDataStreamParams(dataStream); - const [createdOn, [dataStreamInfo], datasetUserPrivileges] = await Promise.all([ getDataStreamCreatedOn(esClient, dataStream), dataStreamService.getMatchingDataStreams(esClient, dataStream), @@ -39,12 +37,14 @@ export async function getDataStreamSettings({ const integration = dataStreamInfo?._meta?.package?.name; const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0]; + const indexTemplate = dataStreamInfo?.template; return { createdOn, integration, datasetUserPrivileges, lastBackingIndexName: lastBackingIndex?.index_name, + indexTemplate, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts index 433086c0b3e52..cbaa637dc60bc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts @@ -13,6 +13,7 @@ export interface DataStreamSettingResponse { totalFieldLimit: number; ignoreDynamicBeyondLimit?: boolean; ignoreMalformed?: boolean; + defaultPipeline?: string; } const DEFAULT_FIELD_LIMIT = 1000; @@ -28,16 +29,20 @@ export async function getDataStreamSettings({ lastBackingIndex: string; }): Promise { const settings = await datasetQualityESClient.settings({ index: dataStream }); - const indexSettings = settings[lastBackingIndex]?.settings?.index?.mapping; + const setting = settings[lastBackingIndex]?.settings; + const mappingsInsideSettings = setting?.index?.mapping; return { - nestedFieldLimit: indexSettings?.nested_fields?.limit - ? Number(indexSettings?.nested_fields?.limit) + nestedFieldLimit: mappingsInsideSettings?.nested_fields?.limit + ? Number(mappingsInsideSettings?.nested_fields?.limit) : DEFAULT_NESTED_FIELD_LIMIT, - totalFieldLimit: indexSettings?.total_fields?.limit - ? Number(indexSettings?.total_fields?.limit) + totalFieldLimit: mappingsInsideSettings?.total_fields?.limit + ? Number(mappingsInsideSettings?.total_fields?.limit) : DEFAULT_FIELD_LIMIT, - ignoreDynamicBeyondLimit: toBoolean(indexSettings?.total_fields?.ignore_dynamic_beyond_limit), - ignoreMalformed: toBoolean(indexSettings?.ignore_malformed), + ignoreDynamicBeyondLimit: toBoolean( + mappingsInsideSettings?.total_fields?.ignore_dynamic_beyond_limit + ), + ignoreMalformed: toBoolean(mappingsInsideSettings?.ignore_malformed), + defaultPipeline: setting?.index?.default_pipeline, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts index a0e7606b475b2..97ff0b124aae9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts @@ -28,7 +28,13 @@ export async function analyzeDegradedField({ const [ { fieldCount, fieldPresent, fieldMapping }, - { nestedFieldLimit, totalFieldLimit, ignoreDynamicBeyondLimit, ignoreMalformed }, + { + nestedFieldLimit, + totalFieldLimit, + ignoreDynamicBeyondLimit, + ignoreMalformed, + defaultPipeline, + }, ] = await Promise.all([ getDataStreamMapping({ datasetQualityESClient, @@ -48,5 +54,6 @@ export async function analyzeDegradedField({ totalFieldLimit, ignoreMalformed, nestedFieldLimit, + defaultPipeline, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 047004d58a6a2..41ba3ee8c7299 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -16,6 +16,8 @@ import { DatasetUserPrivileges, DegradedFieldValues, DegradedFieldAnalysis, + UpdateFieldLimitResponse, + DataStreamRolloverResponse, } from '../../../common/api_types'; import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; @@ -29,6 +31,8 @@ import { getDegradedFields } from './get_degraded_fields'; import { getDegradedFieldValues } from './get_degraded_field_values'; import { analyzeDegradedField } from './get_degraded_field_analysis'; import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; +import { updateFieldLimit } from './update_field_limit'; +import { createDatasetQualityESClient } from '../../utils'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -324,6 +328,58 @@ const analyzeDegradedFieldRoute = createDatasetQualityServerRoute({ }, }); +const updateFieldLimitRoute = createDatasetQualityServerRoute({ + endpoint: 'PUT /internal/dataset_quality/data_streams/{dataStream}/update_field_limit', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + body: t.type({ + newFieldLimit: t.number, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const updatedLimitResponse = await updateFieldLimit({ + esClient, + newFieldLimit: params.body.newFieldLimit, + dataStream: params.path.dataStream, + }); + + return updatedLimitResponse; + }, +}); + +const rolloverDataStream = createDatasetQualityServerRoute({ + endpoint: 'POST /internal/dataset_quality/data_streams/{dataStream}/rollover', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const { acknowledged } = await datasetQualityESClient.rollover({ + alias: params.path.dataStream, + }); + + return { acknowledged }; + }, +}); + export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, @@ -334,4 +390,6 @@ export const dataStreamsRouteRepository = { ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, ...analyzeDegradedFieldRoute, + ...updateFieldLimitRoute, + ...rolloverDataStream, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts new file mode 100644 index 0000000000000..f377ea1e2642c --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { badRequest } from '@hapi/boom'; +import { createDatasetQualityESClient } from '../../../utils'; +import { updateComponentTemplate } from './update_component_template'; +import { updateLastBackingIndexSettings } from './update_settings_last_backing_index'; +import { UpdateFieldLimitResponse } from '../../../../common/api_types'; +import { getDataStreamSettings } from '../get_data_stream_details'; + +export async function updateFieldLimit({ + esClient, + newFieldLimit, + dataStream, +}: { + esClient: ElasticsearchClient; + newFieldLimit: number; + dataStream: string; +}): Promise { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const { lastBackingIndexName, indexTemplate } = await getDataStreamSettings({ + esClient, + dataStream, + }); + + if (!lastBackingIndexName || !indexTemplate) { + throw badRequest(`Data stream does not exists. Received value "${dataStream}"`); + } + + const { + acknowledged: isComponentTemplateUpdated, + componentTemplateName, + error: errorUpdatingComponentTemplate, + } = await updateComponentTemplate({ datasetQualityESClient, indexTemplate, newFieldLimit }); + + if (errorUpdatingComponentTemplate) { + throw badRequest(errorUpdatingComponentTemplate); + } + + const { acknowledged: isLatestBackingIndexUpdated, error: errorUpdatingBackingIndex } = + await updateLastBackingIndexSettings({ + datasetQualityESClient, + lastBackingIndex: lastBackingIndexName, + newFieldLimit, + }); + + return { + isComponentTemplateUpdated, + isLatestBackingIndexUpdated, + customComponentTemplateName: componentTemplateName, + error: errorUpdatingBackingIndex, + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts new file mode 100644 index 0000000000000..0bf19410bd6ac --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../common/utils/component_template_name'; + +interface UpdateComponentTemplateResponse { + acknowledged: boolean | undefined; + componentTemplateName: string; + error?: string; +} + +export async function updateComponentTemplate({ + datasetQualityESClient, + indexTemplate, + newFieldLimit, +}: { + datasetQualityESClient: DatasetQualityESClient; + indexTemplate: string; + newFieldLimit: number; +}): Promise { + const newSettings = { + settings: { + 'index.mapping.total_fields.limit': newFieldLimit, + }, + }; + + const customComponentTemplateName = `${getComponentTemplatePrefixFromIndexTemplate( + indexTemplate + )}@custom`; + + try { + const { acknowledged } = await datasetQualityESClient.updateComponentTemplate({ + name: customComponentTemplateName, + template: newSettings, + }); + + return { + acknowledged, + componentTemplateName: customComponentTemplateName, + }; + } catch (error) { + return { + acknowledged: undefined, // acknowledge is undefined when the request fails + componentTemplateName: customComponentTemplateName, + error: error.message, + }; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts new file mode 100644 index 0000000000000..b98a315547554 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; + +interface UpdateLastBackingIndexSettingsResponse { + acknowledged: boolean | undefined; + error?: string; +} + +export async function updateLastBackingIndexSettings({ + datasetQualityESClient, + lastBackingIndex, + newFieldLimit, +}: { + datasetQualityESClient: DatasetQualityESClient; + lastBackingIndex: string; + newFieldLimit: number; +}): Promise { + const newSettings = { + 'index.mapping.total_fields.limit': newFieldLimit, + }; + + try { + const { acknowledged } = await datasetQualityESClient.updateSettings({ + index: lastBackingIndex, + settings: newSettings, + }); + + return { acknowledged }; + } catch (error) { + return { + acknowledged: undefined, // acknowledge is undefined when the request fails + error: error.message, + }; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index baa2403690fd8..8a78b4163da95 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -8,11 +8,16 @@ import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { ElasticsearchClient } from '@kbn/core/server'; import { + ClusterPutComponentTemplateRequest, + ClusterPutComponentTemplateResponse, FieldCapsRequest, FieldCapsResponse, Indices, IndicesGetMappingResponse, IndicesGetSettingsResponse, + IndicesPutSettingsRequest, + IndicesPutSettingsResponse, + IndicesRolloverResponse, } from '@elastic/elasticsearch/lib/api/types'; type DatasetQualityESSearchParams = ESSearchRequest & { @@ -23,12 +28,12 @@ export type DatasetQualityESClient = ReturnType( + search( searchParams: TParams ): Promise> { return esClient.search(searchParams) as Promise; }, - async msearch( + msearch( index = {} as { index?: Indices }, searches: TParams[] ): Promise<{ @@ -38,14 +43,25 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) { searches: searches.map((search) => [index, search]).flat(), }) as Promise; }, - async fieldCaps(params: FieldCapsRequest): Promise { - return esClient.fieldCaps(params) as Promise; + fieldCaps(params: FieldCapsRequest): Promise { + return esClient.fieldCaps(params); }, - async mappings(params: { index: string }): Promise { + mappings(params: { index: string }): Promise { return esClient.indices.getMapping(params); }, - async settings(params: { index: string }): Promise { + settings(params: { index: string }): Promise { return esClient.indices.getSettings(params); }, + updateComponentTemplate( + params: ClusterPutComponentTemplateRequest + ): Promise { + return esClient.cluster.putComponentTemplate(params); + }, + updateSettings(params: IndicesPutSettingsRequest): Promise { + return esClient.indices.putSettings(params); + }, + rollover(params: { alias: string }): Promise { + return esClient.indices.rollover(params); + }, }; } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts new file mode 100644 index 0000000000000..d11d93d9eae51 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const start = '2024-10-17T11:00:00.000Z'; + const end = '2024-10-17T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials.post( + `/internal/dataset_quality/data_streams/${dataStream}/rollover` + ); + } + + describe('Datastream Rollover', function () { + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); + + it('returns acknowledged when rollover is successful', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + expect(resp.body.acknowledged).to.be(true); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts new file mode 100644 index 0000000000000..16c463e8caf6c --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { + createBackingIndexNameWithoutVersion, + getDataStreamSettingsOfEarliestIndex, + rolloverDataStream, +} from './utils/es_utils'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const esClient = getService('es'); + const packageApi = getService('packageApi'); + const config = getService('config'); + const isServerless = !!config.get('serverless'); + const start = '2024-09-20T11:00:00.000Z'; + const end = '2024-09-20T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const syntheticsDataset = 'synthetics'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + const syntheticsDataStreamName = `${type}-${syntheticsDataset}-${namespace}`; + + const defaultDataStreamPrivileges = { + datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, + }; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials.get( + `/internal/dataset_quality/data_streams/${dataStream}/settings` + ); + } + + describe('Dataset quality settings', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + }); + + after(async () => { + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('returns only privileges if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: nonExistentDataStream, + }, + }); + expect(resp.body).eql(defaultDataStreamPrivileges); + }); + + describe('gets the data stream settings for non integrations', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + after(async () => { + await synthtrace.clean(); + }); + + it('returns "createdOn", "indexTemplate" and "lastBackingIndexName" correctly when available for non integration', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + dataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.indexTemplate).to.be('logs'); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace, + })}-000001` + ); + expect(resp.body.datasetUserPrivileges).to.eql( + defaultDataStreamPrivileges.datasetUserPrivileges + ); + }); + + it('returns "createdOn", "indexTemplate" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, dataStreamName); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + dataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` + ); + expect(resp.body.indexTemplate).to.be('logs'); + }); + }); + + describe('gets the data stream settings for integrations', () => { + before(async () => { + await packageApi.installPackage({ + roleAuthc: adminRoleAuthc, + pkg: syntheticsDataset, + }); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(syntheticsDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + after(async () => { + await synthtrace.clean(); + await packageApi.uninstallPackage({ + roleAuthc: adminRoleAuthc, + pkg: syntheticsDataset, + }); + }); + + it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly when available for integration', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + syntheticsDataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: syntheticsDataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.indexTemplate).to.be('logs'); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: syntheticsDataset, + namespace, + })}-000001` + ); + expect(resp.body.datasetUserPrivileges).to.eql( + defaultDataStreamPrivileges.datasetUserPrivileges + ); + }); + + it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, syntheticsDataStreamName); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + syntheticsDataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: syntheticsDataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: syntheticsDataset, + namespace, + })}-000002` + ); + expect(resp.body.indexTemplate).to.be('logs'); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts index ed06b81d647fb..e9e2665548764 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts @@ -9,7 +9,7 @@ import expect from '@kbn/expect'; import { log, timerange } from '@kbn/apm-synthtrace-client'; import { SupertestWithRoleScopeType } from '../../../services'; import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; -import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './es_utils'; +import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './utils/es_utils'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; const MORE_THAN_1024_CHARS = diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 0c660dda0a445..7e555b7a310e1 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -11,5 +11,8 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) describe('Dataset quality', () => { loadTestFile(require.resolve('./integrations')); loadTestFile(require.resolve('./degraded_field_analyze')); + loadTestFile(require.resolve('./data_stream_settings')); + loadTestFile(require.resolve('./data_stream_rollover')); + loadTestFile(require.resolve('./update_field_limit')); }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts index 910dd84bb309e..33b3fccbea8a1 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts @@ -5,15 +5,15 @@ * 2.0. */ -import { RoleCredentials, InternalRequestHeader } from '@kbn/ftr-common-functional-services'; import expect from '@kbn/expect'; import { APIReturnType } from '@kbn/dataset-quality-plugin/common/rest'; import { CustomIntegration } from '../../../services/package_api'; import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { const samlAuth = getService('samlAuth'); - const supertestWithoutAuth = getService('supertestWithoutAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); const packageApi = getService('packageApi'); const endpoint = 'GET /internal/dataset_quality/integrations'; @@ -33,27 +33,30 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { ]; async function callApiAs({ - roleAuthc, - headers, + roleScopedSupertestWithCookieCredentials, }: { - roleAuthc: RoleCredentials; - headers: InternalRequestHeader; + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; }): Promise { - const { body } = await supertestWithoutAuth - .get('/internal/dataset_quality/integrations') - .set(roleAuthc.apiKeyHeader) - .set(headers); + const { body } = await roleScopedSupertestWithCookieCredentials.get( + '/internal/dataset_quality/integrations' + ); return body; } describe('Integrations', () => { let adminRoleAuthc: RoleCredentials; - let internalHeaders: InternalRequestHeader; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; before(async () => { adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); - internalHeaders = samlAuth.getInternalRequestHeader(); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); }); after(async () => { @@ -74,8 +77,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { it('returns all installed integrations and its datasets map', async () => { const body = await callApiAs({ - roleAuthc: adminRoleAuthc, - headers: internalHeaders, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, }); expect(body.integrations.map((integration: Integration) => integration.name)).to.eql([ @@ -108,8 +110,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { it('returns custom integrations and its datasets map', async () => { const body = await callApiAs({ - roleAuthc: adminRoleAuthc, - headers: internalHeaders, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, }); expect(body.integrations.map((integration: Integration) => integration.name)).to.eql([ diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts new file mode 100644 index 0000000000000..3f842cd43f3e2 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { log, timerange } from '@kbn/apm-synthtrace-client'; + +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; +import { createBackingIndexNameWithoutVersion, rolloverDataStream } from './utils/es_utils'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const esClient = getService('es'); + const packageApi = getService('packageApi'); + const start = '2024-10-17T11:00:00.000Z'; + const end = '2024-10-17T11:01:00.000Z'; + const type = 'logs'; + const invalidDataset = 'invalid'; + const integrationsDataset = 'nginx.access'; + const pkg = 'nginx'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const invalidDataStreamName = `${type}-${invalidDataset}-${namespace}`; + const integrationsDataStreamName = `${type}-${integrationsDataset}-${namespace}`; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream, fieldLimit }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + fieldLimit: number; + }; + }) { + return roleScopedSupertestWithCookieCredentials + .put(`/internal/dataset_quality/data_streams/${dataStream}/update_field_limit`) + .send({ + newFieldLimit: fieldLimit, + }); + } + + describe('Update field limit', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + await packageApi.installPackage({ + roleAuthc: adminRoleAuthc, + pkg, + }); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(integrationsDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + after(async () => { + await synthtrace.clean(); + await packageApi.uninstallPackage({ + roleAuthc: adminRoleAuthc, + pkg, + }); + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('should handles failure gracefully when invalid datastream provided ', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: invalidDataStreamName, + fieldLimit: 10, + }, + }); + + expect(resp.body.statusCode).to.be(400); + expect(resp.body.message).to.be( + `Data stream does not exists. Received value "${invalidDataStreamName}"` + ); + }); + + it('should update last backing index and custom component template', async () => { + // We rollover the data stream to create a new backing index + await rolloverDataStream(esClient, integrationsDataStreamName); + + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: integrationsDataStreamName, + fieldLimit: 50, + }, + }); + + expect(resp.body.isComponentTemplateUpdated).to.be(true); + expect(resp.body.isLatestBackingIndexUpdated).to.be(true); + expect(resp.body.customComponentTemplateName).to.be(`${type}-${integrationsDataset}@custom`); + expect(resp.body.error).to.be(undefined); + + const { component_templates: componentTemplates } = + await esClient.cluster.getComponentTemplate({ + name: `${type}-${integrationsDataset}@custom`, + }); + + const customTemplate = componentTemplates.filter( + (tmp) => tmp.name === `${type}-${integrationsDataset}@custom` + ); + + expect(customTemplate).to.have.length(1); + expect( + customTemplate[0].component_template.template.settings?.index?.mapping?.total_fields?.limit + ).to.be('50'); + + const settingsForAllIndices = await esClient.indices.getSettings({ + index: integrationsDataStreamName, + }); + + const backingIndexWithoutVersion = createBackingIndexNameWithoutVersion({ + type, + dataset: integrationsDataset, + namespace, + }); + const settingsForLastBackingIndex = + settingsForAllIndices[backingIndexWithoutVersion + '-000002'].settings; + const settingsForPreviousBackingIndex = + settingsForAllIndices[backingIndexWithoutVersion + '-000001'].settings; + + // Only the Last Backing Index should have the updated limit and not the one previous to it + expect(settingsForLastBackingIndex?.index?.mapping?.total_fields?.limit).to.be('50'); + + // The previous one should have the default limit of 1000 + expect(settingsForPreviousBackingIndex?.index?.mapping?.total_fields?.limit).to.be('1000'); + + // Rollover to test custom component template + await rolloverDataStream(esClient, integrationsDataStreamName); + + const settingsForLatestBackingIndex = await esClient.indices.getSettings({ + index: backingIndexWithoutVersion + '-000003', + }); + + // The new backing index should read settings from custom component template + expect( + settingsForLatestBackingIndex[backingIndexWithoutVersion + '-000003'].settings?.index + ?.mapping?.total_fields?.limit + ).to.be('50'); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts similarity index 63% rename from x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts index 0e041781122cd..a2fa712ba3be7 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts @@ -39,3 +39,21 @@ export async function setDataStreamSettings( settings, }); } + +export async function rolloverDataStream(es: Client, name: string) { + return es.indices.rollover({ alias: name }); +} + +export async function getDataStreamSettingsOfEarliestIndex(es: Client, name: string) { + const matchingIndexesObj = await es.indices.getSettings({ index: name }); + + const matchingIndexes = Object.keys(matchingIndexesObj ?? {}); + matchingIndexes.sort((a, b) => { + return ( + Number(matchingIndexesObj[a].settings?.index?.creation_date) - + Number(matchingIndexesObj[b].settings?.index?.creation_date) + ); + }); + + return matchingIndexesObj[matchingIndexes[0]].settings; +} diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts deleted file mode 100644 index 45f37b44983aa..0000000000000 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { log, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; -import { DatasetQualityApiClientKey } from '../../common/config'; -import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { - expectToReject, - getDataStreamSettingsOfEarliestIndex, - rolloverDataStream, -} from '../../utils'; -import { createBackingIndexNameWithoutVersion } from './es_utils'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const synthtrace = getService('logSynthtraceEsClient'); - const esClient = getService('es'); - const datasetQualityApiClient = getService('datasetQualityApiClient'); - const pkgService = getService('packageService'); - const start = '2023-12-11T18:00:00.000Z'; - const end = '2023-12-11T18:01:00.000Z'; - const type = 'logs'; - const dataset = 'synth.1'; - const integrationDataset = 'apache.access'; - const namespace = 'default'; - const serviceName = 'my-service'; - const hostName = 'synth-host'; - const pkg = { - name: 'apache', - version: '1.14.0', - }; - - const defaultDataStreamPrivileges = { - datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, - }; - - async function callApiAs(user: DatasetQualityApiClientKey, dataStream: string) { - return await datasetQualityApiClient[user]({ - endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', - params: { - path: { - dataStream, - }, - }, - }); - } - - registry.when('DataStream Settings', { config: 'basic' }, () => { - describe('gets the data stream settings', () => { - before(async () => { - // Install Integration and ingest logs for it - await pkgService.installPackage(pkg); - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(integrationDataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - // Ingest basic logs - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(dataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - }); - - it('returns error when dataStream param is not provided', async () => { - const expectedMessage = 'Data Stream name cannot be empty'; - const err = await expectToReject(() => - callApiAs('datasetQualityMonitorUser', encodeURIComponent(' ')) - ); - expect(err.res.status).to.be(400); - expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); - }); - - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApiAs('datasetQualityMonitorUser', nonExistentDataStream); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - - it('returns "createdOn", "integration" and "lastBackingIndexName" correctly when available', async () => { - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${integrationDataset}-${namespace}` - ); - const resp = await callApiAs( - 'datasetQualityMonitorUser', - `${type}-${integrationDataset}-${namespace}` - ); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.integration).to.be('apache'); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ - type, - dataset: integrationDataset, - namespace, - })}-000001` - ); - expect(resp.body.datasetUserPrivileges).to.eql( - defaultDataStreamPrivileges.datasetUserPrivileges - ); - }); - - it('returns "createdOn" and "lastBackingIndexName" for rolled over dataStream', async () => { - await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApiAs( - 'datasetQualityMonitorUser', - `${type}-${dataset}-${namespace}` - ); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` - ); - }); - - after(async () => { - await synthtrace.clean(); - await pkgService.uninstallPackage(pkg); - }); - }); - }); -} diff --git a/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts b/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts new file mode 100644 index 0000000000000..210d5fd349880 --- /dev/null +++ b/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; + +export const logsNginxMappings = (dataset: string): MappingTypeMapping => ({ + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + cloud: { + properties: { + image: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + value: dataset, + }, + namespace: { + type: 'constant_keyword', + value: 'default', + }, + type: { + type: 'constant_keyword', + value: 'logs', + }, + }, + }, + ecs: { + properties: { + version: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + error: { + properties: { + message: { + type: 'match_only_text', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + module: { + type: 'constant_keyword', + value: 'nginx', + }, + }, + }, + host: { + properties: { + containerized: { + type: 'boolean', + }, + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + os: { + properties: { + build: { + type: 'keyword', + ignore_above: 1024, + }, + codename: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + input: { + properties: { + type: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + log: { + properties: { + level: { + type: 'keyword', + ignore_above: 1024, + }, + offset: { + type: 'long', + }, + }, + }, + network: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + nginx: { + properties: { + access: { + properties: { + remote_ip_list: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + }, + }, + test_field: { + type: 'keyword', + ignore_above: 1024, + }, + tls: { + properties: { + established: { + type: 'boolean', + }, + }, + }, + trace: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, +}); diff --git a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts index 2506201aa3b85..1b477644feca0 100644 --- a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts @@ -17,6 +17,7 @@ import { MORE_THAN_1024_CHARS, } from './data'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; +import { logsNginxMappings } from './custom_mappings/custom_integration_mappings'; export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { const PageObjects = getPageObjects([ @@ -30,34 +31,42 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const esClient = getService('es'); const retry = getService('retry'); const to = new Date().toISOString(); + const type = 'logs'; const degradedDatasetName = 'synth.degraded'; - const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const degradedDataStreamName = `${type}-${degradedDatasetName}-${defaultNamespace}`; const degradedDatasetWithLimitsName = 'synth.degraded.rca'; - const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`; + const degradedDatasetWithLimitDataStreamName = `${type}-${degradedDatasetWithLimitsName}-${defaultNamespace}`; const serviceName = 'test_service'; const count = 5; const customComponentTemplateName = 'logs-synth@mappings'; - describe('Degraded fields flyout', () => { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - }); - - after(async () => { - await synthtrace.clean(); - }); + const nginxAccessDatasetName = 'nginx.access'; + const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; + const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; + const nginxPkg = { + name: 'nginx', + version: '1.23.0', + }; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); it('should open and close the flyout when user clicks on the expand button', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, @@ -90,31 +99,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); }); - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - - describe('testing root cause for ignored fields', () => { + describe('detecting root cause for ignored fields', () => { before(async () => { // Create custom component template await synthtrace.createComponentTemplate( @@ -142,8 +127,18 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid hidden: false, }, }); - // Ingest Degraded Logs with 25 fields + + // Install Nginx Integration and ingest logs for it + await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); + + // Create custom component template to avoid issues with LogsDB + await synthtrace.createComponentTemplate( + customComponentTemplateNameNginx, + logsNginxMappings(nginxAccessDatasetName) + ); + await synthtrace.index([ + // Ingest Degraded Logs with 25 fields in degraded DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -161,7 +156,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .defaults({ 'service.name': serviceName, 'trace.id': generateShortId(), - test_field: [MORE_THAN_1024_CHARS, 'hello world'], + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 42 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], }) .timestamp(timestamp) ); @@ -176,8 +194,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid } ); - // Ingest Degraded Logs with 26 field + // Set Limit of 42 + await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { + 'mapping.total_fields.limit': 42, + }); + await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -196,7 +219,31 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -205,9 +252,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: degradedDatasetWithLimitsName, + }) + '-000002', + { + 'mapping.total_fields.limit': 26, + } + ); + + // Set Limit of 43 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: nginxAccessDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 43, + } + ); - // Ingest docs with 26 fields again await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -223,11 +291,34 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .service(serviceName) .namespace(defaultNamespace) .defaults({ - 'log.file.path': '/my-service.log', 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -235,8 +326,128 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ]); }); - describe('field character limit exceeded', () => { - it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(4); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(3); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + describe('character limit exceeded', () => { + it('should display cause as "field character limit exceeded" when a field is ignored due to character limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', @@ -253,25 +464,167 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.closeFlyout(); }); - it('should display values when cause is "field ignored"', async () => { + it('should display values when cause is "field character limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', }); await retry.tryForTime(5000, async () => { - const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + const testFieldValue1Exists = await PageObjects.datasetQuality.doesTextExist( 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', MORE_THAN_1024_CHARS ); - expect(testFieldValueExists).to.be(true); + const testFieldValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + expect(testFieldValue1Exists).to.be(true); + expect(testFieldValue2Exists).to.be(true); }); await PageObjects.datasetQuality.closeFlyout(); }); + + it('should display the maximum character limit when cause is "field character limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const limitValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-characterLimit', + '1024' + ); + expect(limitValueExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should show possible mitigation section with manual options for non integrations', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + // Should point to index template with the datastream name as value + expect(componentTemplateUrl).to.be( + `/data/index_management/templates/${degradedDatasetWithLimitDataStreamName}` + ); + + const nonIntegrationCustomName = `${type}@custom`; + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(nonIntegrationCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + nonIntegrationCustomName + )}` + ); + }); + + it('should show possible mitigation section with different manual options for integrations', async () => { + // Navigate to Integration Dataset + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + const integrationSpecificCustomName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to component template with @custom as value + expect(componentTemplateUrl).to.be( + `/data/index_management/component_templates/${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(integrationSpecificCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + }); }); - describe('field limit exceeded', () => { + describe('past field limit exceeded', () => { it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -289,7 +642,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.closeFlyout(); }); - it('should display the limit when the cause is "field limit exceeded"', async () => { + it('should display the current field limit when the cause is "field limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'cloud', @@ -319,122 +672,194 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); }); - describe('current quality issues', () => { - it('should display issues only from latest backing index when current issues toggle is on', async () => { + describe('current field limit issues', () => { + it('should display increase field limit as a possible mitigation for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); - - expect(currentIssuesToggleState).to.be(false); - - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - - expect(rows.length).to.eql(3); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); - const newCurrentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - expect(newCurrentIssuesToggleState).to.be(true); + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - const newRows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + const linkURL = await linkButton.getAttribute('href'); - expect(newRows.length).to.eql(2); + expect(linkURL).to.be( + 'https://www.elastic.co/guide/en/elasticsearch/reference/master/mapping-settings-limit.html' + ); }); - it('should keep the toggle on when url state says so', async () => { + it('should display increase field limit as a possible mitigation for non integration', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + expandedDegradedField: 'cloud.project', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - expect(currentIssuesToggleState).to.be(true); + // Should not display the panel to increase field limit + await testSubjects.missingOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); + + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); }); - it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + it('should display additional input fields and button increasing the limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Check value in Table - const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const countColumn = table['Docs count']; - expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + // Should display current field limit + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingCurrentLimitFieldText'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const countValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '5' - ); - expect(countValue).to.be(true); - }); + const currentFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingCurrentLimitFieldText' + ); - // Toggle the switch - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const currentFieldLimitValue = await currentFieldLimitInput.getAttribute('value'); + const currentFieldLimit = parseInt(currentFieldLimitValue as string, 10); + const currentFieldLimitDisabledStatus = await currentFieldLimitInput.getAttribute( + 'disabled' ); - // Check value in Table - const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const newCountColumn = newTable['Docs count']; - expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + expect(currentFieldLimit).to.be(43); + expect(currentFieldLimitDisabledStatus).to.be('true'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const newCountValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '15' - ); - expect(newCountValue).to.be(true); - }); + // Should display new field limit + await testSubjects.existOrFail( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitValue = await newFieldLimitInput.getAttribute('value'); + const newFieldLimit = parseInt(newFieldLimitValue as string, 10); + + // Should be 30% more the current limit + const newLimit = Math.round(currentFieldLimit * 1.3); + expect(newFieldLimit).to.be(newLimit); + + // Should display the apply button + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be(null); }); - it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + it('should validate input for new field limit', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + // Should not allow values less than current limit of 43 + await testSubjects.setValue( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText', + '42', + { + clearWithKeyboard: true, + typeCharByChar: true, + } + ); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be('true'); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' ); + const invalidStatus = await newFieldLimitInput.getAttribute('aria-invalid'); + + expect(invalidStatus).to.be('true'); }); - it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + it('should let user increase the field limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + await applyButton.click(); + + await retry.tryForTime(5000, async () => { + // Should display the success callout + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetSuccessCallout' + ); + + // Should display link to component template edited + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + + const ctLink = await testSubjects.find( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + const ctLinkURL = await ctLink.getAttribute('href'); + + const componentTemplateName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to the component template page + expect( + ctLinkURL?.endsWith( + `/data/index_management/component_templates/${encodeURIComponent( + componentTemplateName + )}` + ) + ).to.be(true); + }); + + // Refresh the time range to get the latest data + await PageObjects.datasetQuality.refreshDetailsPageData(); + + // The page should now handle this as ignore_malformed issue and show a warning + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist' ); + // Should not display the panel to increase field limit await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); }); }); @@ -445,6 +870,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid name: degradedDatasetWithLimitDataStreamName, }); await synthtrace.deleteComponentTemplate(customComponentTemplateName); + await PageObjects.observabilityLogsExplorer.uninstallPackage(nginxPkg); + await synthtrace.deleteComponentTemplate(customComponentTemplateNameNginx); }); }); }); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index ccd48e220064a..cef881fe0797c 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -204,6 +204,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv } }, + async waitUntilPossibleMitigationsLoaded() { + await find.waitForDeletedByCssSelector('.euiFlyoutBody .euiSkeletonRectangle', 20 * 1000); + }, + async waitUntilDegradedFieldFlyoutLoaded() { await testSubjects.existOrFail(testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout); }, @@ -239,6 +243,18 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ); }, + generateBackingIndexNameWithoutVersion({ + type = 'logs', + dataset, + namespace = 'default', + }: { + type?: string; + dataset: string; + namespace?: string; + }) { + return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`; + }, + getDatasetsTable(): Promise { return testSubjects.find(testSubjectSelectors.datasetQualityTable); }, @@ -554,3 +570,12 @@ async function getDatasetTableHeaderTexts(tableWrapper: WebElementWrapper) { headerElementWrappers.map((headerElementWrapper) => headerElementWrapper.getVisibleText()) ); } + +function getCurrentDateFormatted() { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}.${month}.${day}`; +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts deleted file mode 100644 index a132bc01c9720..0000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { log, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; -import type { InternalRequestHeader, RoleCredentials } from '../../../../shared/services'; -import { expectToReject, getDataStreamSettingsOfEarliestIndex, rolloverDataStream } from './utils'; -import { - DatasetQualityApiClient, - DatasetQualityApiError, -} from './common/dataset_quality_api_supertest'; -import { DatasetQualityFtrContextProvider } from './common/services'; -import { createBackingIndexNameWithoutVersion } from './utils'; - -export default function ({ getService }: DatasetQualityFtrContextProvider) { - const datasetQualityApiClient: DatasetQualityApiClient = getService('datasetQualityApiClient'); - const synthtrace = getService('logSynthtraceEsClient'); - const svlCommonApi = getService('svlCommonApi'); - const svlUserManager = getService('svlUserManager'); - const esClient = getService('es'); - const start = '2023-12-11T18:00:00.000Z'; - const end = '2023-12-11T18:01:00.000Z'; - const type = 'logs'; - const dataset = 'nginx.access'; - const namespace = 'default'; - const serviceName = 'my-service'; - const hostName = 'synth-host'; - - const defaultDataStreamPrivileges = { - datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, - }; - - async function callApi( - dataStream: string, - roleAuthc: RoleCredentials, - internalReqHeader: InternalRequestHeader - ) { - return await datasetQualityApiClient.slsUser({ - endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', - params: { - path: { - dataStream, - }, - }, - roleAuthc, - internalReqHeader, - }); - } - - describe('gets the data stream settings', () => { - let roleAuthc: RoleCredentials; - let internalReqHeader: InternalRequestHeader; - before(async () => { - roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('admin'); - internalReqHeader = svlCommonApi.getInternalRequestHeader(); - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(dataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - }); - - after(async () => { - await synthtrace.clean(); - await svlUserManager.invalidateM2mApiKeyWithRoleScope(roleAuthc); - }); - - it('returns error when dataStream param is not provided', async () => { - const expectedMessage = 'Data Stream name cannot be empty'; - const err = await expectToReject(() => - callApi(encodeURIComponent(' '), roleAuthc, internalReqHeader) - ); - expect(err.res.status).to.be(400); - expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); - }); - - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApi(nonExistentDataStream, roleAuthc, internalReqHeader); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - - it('returns "createdOn" and "lastBackingIndexName" correctly', async () => { - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ - type, - dataset, - namespace, - })}-000001` - ); - }); - - it('returns "createdOn" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { - await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` - ); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts index 86c8527139846..39b6a3cb476c1 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts @@ -9,7 +9,6 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Dataset Quality', function () { loadTestFile(require.resolve('./data_stream_details')); - loadTestFile(require.resolve('./data_stream_settings')); loadTestFile(require.resolve('./degraded_field_values')); }); } diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts new file mode 100644 index 0000000000000..210d5fd349880 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; + +export const logsNginxMappings = (dataset: string): MappingTypeMapping => ({ + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + cloud: { + properties: { + image: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + value: dataset, + }, + namespace: { + type: 'constant_keyword', + value: 'default', + }, + type: { + type: 'constant_keyword', + value: 'logs', + }, + }, + }, + ecs: { + properties: { + version: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + error: { + properties: { + message: { + type: 'match_only_text', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + module: { + type: 'constant_keyword', + value: 'nginx', + }, + }, + }, + host: { + properties: { + containerized: { + type: 'boolean', + }, + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + os: { + properties: { + build: { + type: 'keyword', + ignore_above: 1024, + }, + codename: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + input: { + properties: { + type: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + log: { + properties: { + level: { + type: 'keyword', + ignore_above: 1024, + }, + offset: { + type: 'long', + }, + }, + }, + network: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + nginx: { + properties: { + access: { + properties: { + remote_ip_list: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + }, + }, + test_field: { + type: 'keyword', + ignore_above: 1024, + }, + tls: { + properties: { + established: { + type: 'boolean', + }, + }, + }, + trace: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, +}); diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts index 712ed11a28f93..0d8d8e8865d52 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts @@ -385,6 +385,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataStream: degradedDataStreamName, }); + await PageObjects.datasetQuality.waitUntilTableLoaded(); + const rows = await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 4072dcec8a25c..59d58a3e83151 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -17,6 +17,7 @@ import { } from './data'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; +import { logsNginxMappings } from './custom_mappings/custom_integration_mappings'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects([ @@ -31,35 +32,43 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esClient = getService('es'); const retry = getService('retry'); const to = new Date().toISOString(); + const type = 'logs'; const degradedDatasetName = 'synth.degraded'; - const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const degradedDataStreamName = `${type}-${degradedDatasetName}-${defaultNamespace}`; const degradedDatasetWithLimitsName = 'synth.degraded.rca'; - const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`; + const degradedDatasetWithLimitDataStreamName = `${type}-${degradedDatasetWithLimitsName}-${defaultNamespace}`; const serviceName = 'test_service'; const count = 5; const customComponentTemplateName = 'logs-synth@mappings'; - describe('Degraded fields flyout', function () { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - await PageObjects.svlCommonPage.loginWithPrivilegedRole(); - }); - - after(async () => { - await synthtrace.clean(); - }); + const nginxAccessDatasetName = 'nginx.access'; + const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; + const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; + const nginxPkg = { + name: 'nginx', + version: '1.23.0', + }; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + await PageObjects.svlCommonPage.loginAsAdmin(); + }); + + after(async () => { + await synthtrace.clean(); + }); it('should open and close the flyout when user clicks on the expand button', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, @@ -88,30 +97,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - describe('testing root cause for ignored fields', () => { before(async () => { // Create custom component template @@ -140,8 +125,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { hidden: false, }, }); - // Ingest Degraded Logs with 25 fields + + // Install Nginx Integration and ingest logs for it + await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); + + // Create custom component template to avoid issues with LogsDB + await synthtrace.createComponentTemplate( + customComponentTemplateNameNginx, + logsNginxMappings(nginxAccessDatasetName) + ); + await synthtrace.index([ + // Ingest Degraded Logs with 25 fields timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -159,7 +154,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .defaults({ 'service.name': serviceName, 'trace.id': generateShortId(), - test_field: [MORE_THAN_1024_CHARS, 'hello world'], + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], }) .timestamp(timestamp) ); @@ -174,8 +192,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); - // Ingest Degraded Logs with 26 field + // Set Limit of 42 + await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { + 'mapping.total_fields.limit': 43, + }); + await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -194,7 +217,31 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 44 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -203,9 +250,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: degradedDatasetWithLimitsName, + }) + '-000002', + { + 'mapping.total_fields.limit': 26, + } + ); + + // Set Limit of 44 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: nginxAccessDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 44, + } + ); - // Ingest docs with 26 fields again await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -221,20 +289,164 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .service(serviceName) .namespace(defaultNamespace) .defaults({ - 'log.file.path': '/my-service.log', 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); }), ]); + await PageObjects.svlCommonPage.loginAsAdmin(); }); - describe('field character limit exceeded', () => { - it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(4); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(3); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + describe('character limit exceeded', () => { + it('should display cause as "field character limit exceeded" when a field is ignored due to character limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', @@ -251,25 +463,169 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.closeFlyout(); }); - it('should display values when cause is "field ignored"', async () => { + it('should display values when cause is "field character limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', }); await retry.tryForTime(5000, async () => { - const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + const testFieldValue1Exists = await PageObjects.datasetQuality.doesTextExist( 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', MORE_THAN_1024_CHARS ); - expect(testFieldValueExists).to.be(true); + const testFieldValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + expect(testFieldValue1Exists).to.be(true); + expect(testFieldValue2Exists).to.be(true); }); await PageObjects.datasetQuality.closeFlyout(); }); + + it('should display the maximum character limit when cause is "field character limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const limitValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-characterLimit', + '1024' + ); + expect(limitValueExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should show possible mitigation section with manual options for non integrations', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + // Should point to index template with the datastream name as value + expect(componentTemplateUrl).to.be( + `/data/index_management/templates/${degradedDatasetWithLimitDataStreamName}` + ); + + const nonIntegrationCustomName = `${type}@custom`; + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(nonIntegrationCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + nonIntegrationCustomName + )}` + ); + }); + + it('should show possible mitigation section with different manual options for integrations', async () => { + // Navigate to Integration Dataset + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'test_field', + }); + + await PageObjects.datasetQuality.waitUntilPossibleMitigationsLoaded(); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + const integrationSpecificCustomName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to component template with @custom as value + expect(componentTemplateUrl).to.be( + `/data/index_management/component_templates/${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(integrationSpecificCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + }); }); - describe('field limit exceeded', () => { + describe('past field limit exceeded', () => { it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -287,7 +643,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.closeFlyout(); }); - it('should display the limit when the cause is "field limit exceeded"', async () => { + it('should display the current field limit when the cause is "field limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'cloud', @@ -317,122 +673,216 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('current quality issues', () => { - it('should display issues only from latest backing index when current issues toggle is on', async () => { + describe('current field limit issues', () => { + it('should display increase field limit as a possible mitigation for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - expect(currentIssuesToggleState).to.be(false); + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - expect(rows.length).to.eql(3); + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const linkURL = await linkButton.getAttribute('href'); + + expect(linkURL).to.be( + 'https://www.elastic.co/guide/en/elasticsearch/reference/master/mapping-settings-limit.html' ); + }); - const newCurrentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + it('should display increase field limit as a possible mitigation for non integration', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud.project', + }); - expect(newCurrentIssuesToggleState).to.be(true); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - const newRows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + // Should not display the panel to increase field limit + await testSubjects.missingOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); - expect(newRows.length).to.eql(2); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); }); - it('should keep the toggle on when url state says so', async () => { + it('should display additional input fields and button increasing the limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Should display current field limit + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingCurrentLimitFieldText'); - expect(currentIssuesToggleState).to.be(true); + const currentFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingCurrentLimitFieldText' + ); + + const currentFieldLimitValue = await currentFieldLimitInput.getAttribute('value'); + const currentFieldLimit = parseInt(currentFieldLimitValue as string, 10); + const currentFieldLimitDisabledStatus = await currentFieldLimitInput.getAttribute( + 'disabled' + ); + + expect(currentFieldLimit).to.be(44); + expect(currentFieldLimitDisabledStatus).to.be('true'); + + // Should display new field limit + await testSubjects.existOrFail( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitValue = await newFieldLimitInput.getAttribute('value'); + const newFieldLimit = parseInt(newFieldLimitValue as string, 10); + + // Should be 30% more the current limit + const newLimit = Math.round(currentFieldLimit * 1.3); + expect(newFieldLimit).to.be(newLimit); + + // Should display the apply button + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be(null); }); - it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + it('should validate input for new field limit', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Check value in Table - const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const countColumn = table['Docs count']; - expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + // Should not allow values less than current limit of 44 + await testSubjects.setValue( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText', + '42', + { + clearWithKeyboard: true, + typeCharByChar: true, + } + ); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const countValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '5' - ); - expect(countValue).to.be(true); + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be('true'); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + const invalidStatus = await newFieldLimitInput.getAttribute('aria-invalid'); + + expect(invalidStatus).to.be('true'); + }); + + it('should validate and show error callout when API call fails', async () => { + await PageObjects.svlCommonPage.loginWithPrivilegedRole(); + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Toggle the switch - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - // Check value in Table - const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const newCountColumn = newTable['Docs count']; - expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + await applyButton.click(); - // Check value in Flyout await retry.tryForTime(5000, async () => { - const newCountValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '15' - ); - expect(newCountValue).to.be(true); + // Should display the error callout + await testSubjects.existOrFail('datasetQualityDetailsNewFieldLimitErrorCallout'); }); + + await PageObjects.svlCommonPage.loginAsAdmin(); }); - it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + it('should let user increase the field limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - }); - it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', + await applyButton.click(); + + await retry.tryForTime(5000, async () => { + // Should display the success callout + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetSuccessCallout' + ); + + // Should display link to component template edited + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + + const ctLink = await testSubjects.find( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + const ctLinkURL = await ctLink.getAttribute('href'); + + const componentTemplateName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to the component template page + expect( + ctLinkURL?.endsWith( + `/data/index_management/component_templates/${encodeURIComponent( + componentTemplateName + )}` + ) + ).to.be(true); }); - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout - ); + // Refresh the time range to get the latest data + await PageObjects.datasetQuality.refreshDetailsPageData(); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + // The page should now handle this as ignore_malformed issue and show a warning + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist' ); + // Should not display the panel to increase field limit await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); }); }); @@ -443,6 +893,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { name: degradedDatasetWithLimitDataStreamName, }); await synthtrace.deleteComponentTemplate(customComponentTemplateName); + await PageObjects.observabilityLogsExplorer.uninstallPackage(nginxPkg); + await synthtrace.deleteComponentTemplate(customComponentTemplateNameNginx); }); }); });