diff --git a/x-pack/packages/index-management/index_management_shared_types/src/types.ts b/x-pack/packages/index-management/index_management_shared_types/src/types.ts index 190aadfc29d01..ec5c7938d6b4b 100644 --- a/x-pack/packages/index-management/index_management_shared_types/src/types.ts +++ b/x-pack/packages/index-management/index_management_shared_types/src/types.ts @@ -17,10 +17,21 @@ import type { LocatorPublic } from '@kbn/share-plugin/public'; import { ExtensionsSetup } from './services/extensions_service'; import { PublicApiServiceSetup } from './services/public_api_service'; -export interface IndexManagementLocatorParams extends SerializableRecord { - page: 'data_streams_details'; - dataStreamName?: string; -} +export type IndexManagementLocatorParams = SerializableRecord & + ( + | { + page: 'data_streams_details'; + dataStreamName?: string; + } + | { + page: 'index_template'; + indexTemplate: string; + } + | { + page: 'component_template'; + componentTemplate: string; + } + ); export type IndexManagementLocator = LocatorPublic; diff --git a/x-pack/plugins/index_management/public/application/services/routing.ts b/x-pack/plugins/index_management/public/application/services/routing.ts index bce7a14f03e46..89143bbd79d02 100644 --- a/x-pack/plugins/index_management/public/application/services/routing.ts +++ b/x-pack/plugins/index_management/public/application/services/routing.ts @@ -11,6 +11,7 @@ import { Section } from '../../../common/constants'; import type { IndexDetailsTabId } from '../../../common/constants'; import { ExtensionsService } from '../../services/extensions_service'; import { IndexDetailsSection } from '../../../common/constants'; + export const getTemplateListLink = () => `/templates`; export const getTemplateDetailsLink = (name: string, isLegacy?: boolean) => { @@ -81,6 +82,11 @@ export const getComponentTemplatesLink = (usedByTemplateName?: string) => { } return url; }; + +export const getComponentTemplateDetailLink = (name: string) => { + return `/component_templates/${encodeURIComponent(name)}`; +}; + export const navigateToIndexDetailsPage = ( indexName: string, indicesListURLParams: string, diff --git a/x-pack/plugins/index_management/public/locator.test.ts b/x-pack/plugins/index_management/public/locator.test.ts index 712223d7cbfe4..49b9890259d9c 100644 --- a/x-pack/plugins/index_management/public/locator.test.ts +++ b/x-pack/plugins/index_management/public/locator.test.ts @@ -34,4 +34,26 @@ describe('Index Management URL locator', () => { }); expect(path).toBe('/data/index_management/data_streams/test'); }); + + test('locator returns the correct url for index_template', async () => { + const indexTemplateName = 'test@custom'; + const { path } = await locator.getLocation({ + page: 'index_template', + indexTemplate: indexTemplateName, + }); + expect(path).toBe( + encodeURI(`/data/index_management/templates/${encodeURIComponent(indexTemplateName)}`) + ); + }); + + test('locator returns the correct url for component_template', async () => { + const componentTemplateName = 'log@custom'; + const { path } = await locator.getLocation({ + page: 'component_template', + componentTemplate: componentTemplateName, + }); + expect(path).toBe( + `/data/index_management/component_templates/${encodeURIComponent(componentTemplateName)}` + ); + }); }); diff --git a/x-pack/plugins/index_management/public/locator.ts b/x-pack/plugins/index_management/public/locator.ts index d32d33573507d..c5411aded71a4 100644 --- a/x-pack/plugins/index_management/public/locator.ts +++ b/x-pack/plugins/index_management/public/locator.ts @@ -8,7 +8,11 @@ import { ManagementAppLocator } from '@kbn/management-plugin/common'; import { LocatorDefinition } from '@kbn/share-plugin/public'; import { IndexManagementLocatorParams } from '@kbn/index-management-shared-types'; -import { getDataStreamDetailsLink } from './application/services/routing'; +import { + getComponentTemplateDetailLink, + getDataStreamDetailsLink, + getTemplateDetailsLink, +} from './application/services/routing'; import { PLUGIN } from '../common/constants'; export const INDEX_MANAGEMENT_LOCATOR_ID = 'INDEX_MANAGEMENT_LOCATOR_ID'; @@ -37,6 +41,18 @@ export class IndexManagementLocatorDefinition path: location.path + getDataStreamDetailsLink(params.dataStreamName!), }; } + case 'index_template': { + return { + ...location, + path: location.path + getTemplateDetailsLink(params.indexTemplate), + }; + } + case 'component_template': { + return { + ...location, + path: location.path + getComponentTemplateDetailLink(params.componentTemplate), + }; + } } }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index bfbb2bc1cd5d1..903d7f0607663 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -134,22 +134,39 @@ export const degradedFieldAnalysisRt = rt.intersection([ type: rt.string, ignore_above: rt.number, }), + defaultPipeline: rt.string, }), ]); export type DegradedFieldAnalysis = rt.TypeOf; -export const dataStreamSettingsRt = rt.intersection([ +export const updateFieldLimitResponseRt = rt.intersection([ rt.type({ - lastBackingIndexName: rt.string, + isComponentTemplateUpdated: rt.union([rt.boolean, rt.undefined]), + isLatestBackingIndexUpdated: rt.union([rt.boolean, rt.undefined]), + customComponentTemplateName: rt.string, }), rt.partial({ - createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless - integration: rt.string, - datasetUserPrivileges: datasetUserPrivilegesRt, + error: rt.string, }), ]); +export type UpdateFieldLimitResponse = rt.TypeOf; + +export const dataStreamRolloverResponseRt = rt.type({ + acknowledged: rt.boolean, +}); + +export type DataStreamRolloverResponse = rt.TypeOf; + +export const dataStreamSettingsRt = rt.partial({ + lastBackingIndexName: rt.string, + indexTemplate: rt.string, + createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless + integration: rt.string, + datasetUserPrivileges: datasetUserPrivilegesRt, +}); + export type DataStreamSettings = rt.TypeOf; export const dataStreamDetailsRt = rt.partial({ diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts index 66b7567a2b60c..ce74552b581b9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts @@ -14,3 +14,8 @@ export interface AnalyzeDegradedFieldsParams { lastBackingIndex: string; degradedField: string; } + +export interface UpdateFieldLimitParams { + dataStream: string; + newFieldLimit: number; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index e5b660b31de10..1026dd8ea58d3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -500,3 +500,182 @@ export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate 'This issue was detected in an older version of the dataset, but not in the most recent version.', } ); + +export const possibleMitigationTitle = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigationTitle', + { + defaultMessage: 'Possible mitigation', + } +); + +export const increaseFieldMappingLimitTitle = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.increaseFieldMappingLimitTitle', + { + defaultMessage: 'Increase field mapping limit', + } +); + +export const fieldLimitMitigationDescriptionText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationDescription', + { + defaultMessage: + 'The field mapping limit sets the maximum number of fields in an index. When exceeded, additional fields are ignored. To prevent this, increase your field mapping limit.', + } +); + +export const fieldLimitMitigationConsiderationText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations', + { + defaultMessage: 'Before changing the field limit, consider the following:', + } +); + +export const fieldLimitMitigationConsiderationText1 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations1', + { + defaultMessage: 'Increasing the field limit could slow cluster performance.', + } +); + +export const fieldLimitMitigationConsiderationText2 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations2', + { + defaultMessage: 'Increasing the field limit also resolves field limit issues for other fields.', + } +); + +export const fieldLimitMitigationConsiderationText3 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations3', + { + defaultMessage: + 'This change applies to the [name] component template and affects all namespaces in the template.', + } +); + +export const fieldLimitMitigationConsiderationText4 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations4', + { + defaultMessage: + 'You need to roll over affected data streams to apply mapping changes to component templates.', + } +); + +export const fieldLimitMitigationCurrentLimitLabelText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationCurrentLimitLabelText', + { + defaultMessage: 'Current limit', + } +); + +export const fieldLimitMitigationNewLimitButtonText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitButtonText', + { + defaultMessage: 'New limit', + } +); + +export const fieldLimitMitigationNewLimitPlaceholderText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitPlaceholderText', + { + defaultMessage: 'New field limit', + } +); + +export const fieldLimitMitigationApplyButtonText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationApplyButtonText', + { + defaultMessage: 'Apply', + } +); + +export const otherMitigationsLoadingAriaText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsLoadingText', + { + defaultMessage: 'Loading possible mitigations', + } +); + +export const otherMitigationsCustomComponentTemplate = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomComponentTemplate', + { + defaultMessage: 'Add or edit custom component template', + } +); + +export const otherMitigationsCustomIngestPipeline = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomIngestPipeline', + { + defaultMessage: 'Add or edit custom ingest pipeline', + } +); + +export const fieldLimitMitigationOfficialDocumentation = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationOfficialDocumentation', + { + defaultMessage: 'Documentation', + } +); + +export const fieldLimitMitigationSuccessMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessMessage', + { + defaultMessage: 'New limit set!', + } +); + +export const fieldLimitMitigationSuccessComponentTemplateLinkText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessComponentTemplateLinkText', + { + defaultMessage: 'See component template', + } +); + +export const fieldLimitMitigationPartiallyFailedMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessage', + { + defaultMessage: 'Changes not applied to new data', + } +); + +export const fieldLimitMitigationFailedMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessage', + { + defaultMessage: 'Changes not applied', + } +); + +export const fieldLimitMitigationFailedMessageDescription = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessageDescription', + { + defaultMessage: 'Failed to set new limit', + } +); + +export const fieldLimitMitigationPartiallyFailedMessageDescription = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessageDescription', + { + defaultMessage: + 'The component template was successfully updated with the new field limit, but the changes were not applied to the most recent backing index. Perform a rollover to apply your changes to new data.', + } +); + +export const fieldLimitMitigationRolloverButton = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationRolloverButton', + { + defaultMessage: 'Rollover', + } +); + +export const manualMitigationCustomPipelineCopyPipelineNameAriaText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.copyPipelineNameAriaText', + { + defaultMessage: 'Copy pipeline name', + } +); + +export const manualMitigationCustomPipelineCreateEditPipelineLink = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.createEditPipelineLink', + { + defaultMessage: 'create or edit the pipeline', + } +); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts new file mode 100644 index 0000000000000..0be7b84137c83 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * There are index templates like this metrics-apm.service_transaction.10m@template which exists. + * Hence this @ needs to be removed to derive the custom component template name. + */ +export function getComponentTemplatePrefixFromIndexTemplate(indexTemplate: string) { + if (indexTemplate.includes('@')) { + return indexTemplate.split('@')[0]; + } + + return indexTemplate; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx index 1e6bda781d733..3bcee0bbc89b3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx @@ -38,7 +38,7 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) degradedFieldValues, isDegradedFieldsLoading, isAnalysisInProgress, - degradedFieldAnalysisResult, + degradedFieldAnalysisFormattedResult, degradedFieldAnalysis, } = useDegradedFields(); @@ -94,9 +94,12 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) grow={2} >
- + - {degradedFieldAnalysisResult?.potentialCause} + {degradedFieldAnalysisFormattedResult?.potentialCause}
@@ -125,52 +128,53 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) )} - {!isAnalysisInProgress && degradedFieldAnalysisResult?.shouldDisplayValues && ( - <> - - - - {degradedFieldMaximumCharacterLimitColumnName} - - - + - {degradedFieldAnalysis?.fieldMapping?.ignore_above} - - - - - - - {degradedFieldValuesColumnName} - - - + + {degradedFieldMaximumCharacterLimitColumnName} + + + + {degradedFieldAnalysis?.fieldMapping?.ignore_above} + + + + - - {degradedFieldValues?.values.map((value, idx) => ( - - - {value} - - - ))} - - - - - - )} + + + {degradedFieldValuesColumnName} + + + + + {degradedFieldValues?.values.map((value, idx) => ( + + + {value} + + + ))} + + + + + + )} ); }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx index 189b3ceefe37c..bb72b4f6de20f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx @@ -6,6 +6,7 @@ */ import React, { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiBadge, EuiFlyout, @@ -20,6 +21,7 @@ import { EuiButtonIcon, EuiToolTip, } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; import { NavigationSource } from '../../../services/telemetry'; import { useDatasetDetailsRedirectLinkTelemetry, @@ -38,11 +40,18 @@ import { } from '../../../../common/translations'; import { DegradedFieldInfo } from './field_info'; import { _IGNORED } from '../../../../common/es_fields'; +import { PossibleMitigations } from './possible_mitigations'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DegradedFieldFlyout() { - const { closeDegradedFieldFlyout, expandedDegradedField, renderedItems } = useDegradedFields(); + const { + closeDegradedFieldFlyout, + expandedDegradedField, + renderedItems, + isAnalysisInProgress, + degradedFieldAnalysisFormattedResult, + } = useDegradedFields(); const { dataStreamSettings, datasetDetails, timeRange } = useDatasetQualityDetailsState(); const pushedFlyoutTitleId = useGeneratedHtmlId({ prefix: 'pushedFlyoutTitle', @@ -118,9 +127,42 @@ export default function DegradedFieldFlyout() { )} + {isUserViewingTheIssueOnLatestBackingIndex && + !isAnalysisInProgress && + degradedFieldAnalysisFormattedResult && + !degradedFieldAnalysisFormattedResult.identifiedUsingHeuristics && ( + <> + + + + {i18n.translate( + 'xpack.datasetQuality.degradedFieldFlyout.strong.fieldLimitLabel', + { defaultMessage: 'field limit' } + )} + + ), + }} + /> + + + )} + {isUserViewingTheIssueOnLatestBackingIndex && ( + <> + + + + )} ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx new file mode 100644 index 0000000000000..0dd80bb120e54 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiLink } from '@elastic/eui'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../common/translations'; + +export function FieldLimitDocLink() { + const { + services: { docLinks }, + } = useKibanaContextForPlugin(); + + return ( + + {fieldLimitMitigationOfficialDocumentation} + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx new file mode 100644 index 0000000000000..1056713ac2070 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { + EuiAccordion, + EuiHorizontalRule, + EuiPanel, + EuiSpacer, + EuiText, + EuiTitle, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { + fieldLimitMitigationConsiderationText, + fieldLimitMitigationConsiderationText1, + fieldLimitMitigationConsiderationText2, + fieldLimitMitigationConsiderationText3, + fieldLimitMitigationConsiderationText4, + fieldLimitMitigationDescriptionText, + increaseFieldMappingLimitTitle, +} from '../../../../../../common/translations'; +import { useDegradedFields } from '../../../../../hooks'; +import { IncreaseFieldMappingLimit } from './increase_field_mapping_limit'; +import { FieldLimitDocLink } from './field_limit_documentation_link'; +import { MessageCallout } from './message_callout'; + +export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean }) { + const accordionId = useGeneratedHtmlId({ + prefix: increaseFieldMappingLimitTitle, + }); + + const { degradedFieldAnalysis } = useDegradedFields(); + + const accordionTitle = ( + +
{increaseFieldMappingLimitTitle}
+
+ ); + + return ( + + + + {fieldLimitMitigationDescriptionText} + + + +

{fieldLimitMitigationConsiderationText}

+ +
    +
  • {fieldLimitMitigationConsiderationText1}
  • +
  • {fieldLimitMitigationConsiderationText2}
  • +
  • {fieldLimitMitigationConsiderationText3}
  • +
  • {fieldLimitMitigationConsiderationText4}
  • +
+
+ + {isIntegration && ( + <> + + + + + + )} + +
+
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx new file mode 100644 index 0000000000000..158a5e5eba460 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiFieldText, + EuiFormRow, + EuiButton, + EuiFieldNumber, +} from '@elastic/eui'; +import { + fieldLimitMitigationApplyButtonText, + fieldLimitMitigationCurrentLimitLabelText, + fieldLimitMitigationNewLimitButtonText, + fieldLimitMitigationNewLimitPlaceholderText, +} from '../../../../../../common/translations'; +import { useDegradedFields } from '../../../../../hooks'; + +export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit: number }) { + // Propose the user a 30% increase over the current limit + const proposedNewLimit = Math.round(totalFieldLimit * 1.3); + const [newFieldLimit, setNewFieldLimit] = useState(proposedNewLimit); + const [isInvalid, setIsInvalid] = useState(false); + const { updateNewFieldLimit, isMitigationInProgress } = useDegradedFields(); + + const validateNewLimit = (newLimit: string) => { + const parsedLimit = parseInt(newLimit, 10); + setNewFieldLimit(parsedLimit); + if (totalFieldLimit > parsedLimit) { + setIsInvalid(true); + } else { + setIsInvalid(false); + } + }; + + return ( + + + + + + + + + validateNewLimit(e.target.value)} + aria-label={fieldLimitMitigationNewLimitPlaceholderText} + isInvalid={isInvalid} + min={totalFieldLimit + 1} + /> + + + + + updateNewFieldLimit(newFieldLimit)} + isLoading={isMitigationInProgress} + > + {fieldLimitMitigationApplyButtonText} + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx new file mode 100644 index 0000000000000..168ae4df575e9 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiButton, EuiCallOut, EuiLink } from '@elastic/eui'; +import { + fieldLimitMitigationFailedMessage, + fieldLimitMitigationFailedMessageDescription, + fieldLimitMitigationPartiallyFailedMessage, + fieldLimitMitigationPartiallyFailedMessageDescription, + fieldLimitMitigationRolloverButton, + fieldLimitMitigationSuccessComponentTemplateLinkText, + fieldLimitMitigationSuccessMessage, +} from '../../../../../../common/translations'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; +import { useKibanaContextForPlugin } from '../../../../../utils'; + +export function MessageCallout() { + const { + isMitigationInProgress, + newFieldLimitData, + isRolloverRequired, + isMitigationAppliedSuccessfully, + } = useDegradedFields(); + const { error: serverError } = newFieldLimitData ?? {}; + + if (serverError) { + return ; + } + + if (!isMitigationInProgress && isRolloverRequired) { + return ; + } + + if (!isMitigationInProgress && isMitigationAppliedSuccessfully) { + return ; + } + + return null; +} + +export function SuccessCallout() { + const { + services: { + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); + const { name } = datasetDetails; + + const componentTemplateUrl = locators.get('INDEX_MANAGEMENT_LOCATOR_ID')?.useUrl({ + page: 'component_template', + componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom`, + }); + + return ( + + + {fieldLimitMitigationSuccessComponentTemplateLinkText} + + + ); +} + +export function ManualRolloverCallout() { + const { triggerRollover, isRolloverInProgress } = useDegradedFields(); + return ( + +

{fieldLimitMitigationPartiallyFailedMessageDescription}

+ + {fieldLimitMitigationRolloverButton} + +
+ ); +} + +export function ErrorCallout() { + return ( + +

{fieldLimitMitigationFailedMessageDescription}

+
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx new file mode 100644 index 0000000000000..34f39f25a67ec --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSpacer } from '@elastic/eui'; +import { ManualMitigations } from './manual'; +import { FieldMappingLimit } from './field_limit/field_mapping_limit'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../hooks'; +import { PossibleMitigationTitle } from './title'; + +export function PossibleMitigations() { + const { degradedFieldAnalysis, isAnalysisInProgress } = useDegradedFields(); + const { integrationDetails } = useDatasetQualityDetailsState(); + const isIntegration = Boolean(integrationDetails?.integration); + + return ( + !isAnalysisInProgress && ( +
+ + + {degradedFieldAnalysis?.isFieldLimitIssue && ( + <> + + + + )} + +
+ ) + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx new file mode 100644 index 0000000000000..54bbe91f2f2e1 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants'; +import { EuiFlexGroup, EuiIcon, EuiLink, EuiPanel, EuiTitle } from '@elastic/eui'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; +import { otherMitigationsCustomComponentTemplate } from '../../../../../../common/translations'; + +export function CreateEditComponentTemplateLink({ isIntegration }: { isIntegration: boolean }) { + const { + services: { + application, + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + + const [indexTemplatePath, setIndexTemplatePath] = useState(null); + const [componentTemplatePath, setComponentTemplatePath] = useState(null); + + const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); + const { name } = datasetDetails; + + const indexManagementLocator = locators.get('INDEX_MANAGEMENT_LOCATOR_ID'); + + useEffect(() => { + indexManagementLocator + ?.getLocation({ + page: 'index_template', + indexTemplate: dataStreamSettings?.indexTemplate ?? '', + }) + .then(({ path }) => setIndexTemplatePath(path)); + indexManagementLocator + ?.getLocation({ + page: 'component_template', + componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom`, + }) + .then(({ path }) => setComponentTemplatePath(path)); + }, [ + locators, + setIndexTemplatePath, + dataStreamSettings?.indexTemplate, + indexManagementLocator, + name, + ]); + + const templateUrl = isIntegration ? componentTemplatePath : indexTemplatePath; + + const onClickHandler = useCallback(async () => { + const options = { + openInNewTab: true, + ...(templateUrl && { path: templateUrl }), + }; + + await application.navigateToApp(MANAGEMENT_APP_ID, options); + }, [application, templateUrl]); + + return ( + + + + + +

{otherMitigationsCustomComponentTemplate}

+
+
+
+
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx new file mode 100644 index 0000000000000..f931f3461fb57 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSkeletonRectangle, EuiSpacer } from '@elastic/eui'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; +import { CreateEditComponentTemplateLink } from './component_template_link'; +import { CreateEditPipelineLink } from './pipeline_link'; +import { otherMitigationsLoadingAriaText } from '../../../../../../common/translations'; + +export function ManualMitigations() { + const { integrationDetails, loadingState, dataStreamSettings } = useDatasetQualityDetailsState(); + const isIntegrationPresentInSettings = dataStreamSettings?.integration; + const isIntegration = !!integrationDetails?.integration; + const { dataStreamSettingsLoading, integrationDetailsLoadings } = loadingState; + + const hasIntegrationCheckCompleted = + !dataStreamSettingsLoading && + ((isIntegrationPresentInSettings && !integrationDetailsLoadings) || + !isIntegrationPresentInSettings); + + return ( + + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx new file mode 100644 index 0000000000000..6179a3ed0736c --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useMemo } from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; +import { + copyToClipboard, + EuiAccordion, + EuiButtonIcon, + EuiFieldText, + EuiHorizontalRule, + EuiLink, + EuiPanel, + EuiSpacer, + EuiTitle, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { + manualMitigationCustomPipelineCopyPipelineNameAriaText, + manualMitigationCustomPipelineCreateEditPipelineLink, + otherMitigationsCustomIngestPipeline, +} from '../../../../../../common/translations'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; + +const AccordionTitle = () => ( + +
{otherMitigationsCustomIngestPipeline}
+
+); + +export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boolean }) { + const { + services: { + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + + const accordionId = useGeneratedHtmlId({ + prefix: otherMitigationsCustomIngestPipeline, + }); + + const { datasetDetails } = useDatasetQualityDetailsState(); + const { type, name } = datasetDetails; + + const pipelineName = useMemo( + () => (isIntegration ? `${type}-${name}@custom` : `${type}@custom`), + [isIntegration, type, name] + ); + + const ingestPipelineLocator = locators.get('INGEST_PIPELINES_APP_LOCATOR'); + + const pipelineUrl = ingestPipelineLocator?.useUrl( + { pipelineId: pipelineName, page: 'pipelines_list' }, + {}, + [pipelineName] + ); + + const onClickHandler = useCallback(() => { + copyToClipboard(pipelineName); + }, [pipelineName]); + + return ( + + } + paddingSize="none" + initialIsOpen={true} + data-test-subj="datasetQualityManualMitigationsPipelineAccordion" + > + + + {i18n.translate('xpack.datasetQuality.editPipeline.strong.Label', { + defaultMessage: '1.', + })} + + ), + }} + /> + + + } + readOnly={true} + aria-label={manualMitigationCustomPipelineCopyPipelineNameAriaText} + value={pipelineName} + data-test-subj="datasetQualityManualMitigationsPipelineName" + fullWidth + /> + + + {i18n.translate('xpack.datasetQuality.editPipeline.strong.Label', { + defaultMessage: '2.', + })} + + ), + createEditPipelineLink: ( + + {manualMitigationCustomPipelineCreateEditPipelineLink} + + ), + }} + /> + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx new file mode 100644 index 0000000000000..93e253b0f849c --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiBetaBadge, EuiFlexGroup, EuiIcon, EuiTitle } from '@elastic/eui'; + +import { + overviewQualityIssuesAccordionTechPreviewBadge, + possibleMitigationTitle, +} from '../../../../../common/translations'; + +export function PossibleMitigationTitle() { + return ( + + + +

{possibleMitigationTitle}

+
+ +
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx index b33bd11dbe3a6..0cdc460cd56dc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx @@ -45,6 +45,7 @@ export function DegradedFields() { aria-describedby={toggleTextSwitchId} compressed data-test-subj="datasetQualityDetailsOverviewDegradedFieldToggleSwitch" + css={{ marginRight: '5px' }} /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts index 78ad0e53dd5e2..49ceb50abc3cd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts @@ -104,20 +104,25 @@ export function useDegradedFields() { }, [service]); const degradedFieldValues = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.ignoredValues.done') + state.matches('initializing.degradedFieldFlyout.open.initialized.ignoredValues.done') ? state.context.degradedFieldValues : undefined ); const degradedFieldAnalysis = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.analyze.done') + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating') || + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + ) || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error') ? state.context.degradedFieldAnalysis : undefined ); - // This piece only cater field limit issue at the moment. - // In future this will cater the other 2 reasons as well - const degradedFieldAnalysisResult = useMemo(() => { + const degradedFieldAnalysisFormattedResult = useMemo(() => { if (!degradedFieldAnalysis) { return undefined; } @@ -127,8 +132,8 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldLimitExceeded, tooltipContent: degradedFieldCauseFieldLimitExceededTooltip, - shouldDisplayMitigation: true, - shouldDisplayValues: false, + shouldDisplayIgnoredValuesAndLimit: false, + identifiedUsingHeuristics: true, }; } @@ -143,8 +148,8 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldIgnored, tooltipContent: degradedFieldCauseFieldIgnoredTooltip, - shouldDisplayMitigation: false, - shouldDisplayValues: true, + shouldDisplayIgnoredValuesAndLimit: true, + identifiedUsingHeuristics: true, }; } } @@ -153,19 +158,59 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldMalformed, tooltipContent: degradedFieldCauseFieldMalformedTooltip, - shouldDisplayMitigation: false, - shouldDisplayValues: false, + shouldDisplayIgnoredValuesAndLimit: false, + identifiedUsingHeuristics: false, // TODO: Add heuristics to identify ignore_malformed issues }; }, [degradedFieldAnalysis, degradedFieldValues]); const isDegradedFieldsValueLoading = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.ignoredValues.fetching'); + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' + ); + }); + + const isRolloverRequired = useSelector(service, (state) => { + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + ); + }); + + const isMitigationAppliedSuccessfully = useSelector(service, (state) => { + return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success'); }); const isAnalysisInProgress = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.analyze.fetching'); + return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing'); + }); + + const isRolloverInProgress = useSelector(service, (state) => { + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver' + ); + }); + + const updateNewFieldLimit = useCallback( + (newFieldLimit: number) => { + service.send({ type: 'SET_NEW_FIELD_LIMIT', newFieldLimit }); + }, + [service] + ); + + const isMitigationInProgress = useSelector(service, (state) => { + return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating'); }); + const newFieldLimitData = useSelector(service, (state) => + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') || + state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error') + ? state.context.fieldLimit + : undefined + ); + + const triggerRollover = useCallback(() => { + service.send('ROLLOVER_DATA_STREAM'); + }, [service]); + return { isDegradedFieldsLoading, pagination, @@ -181,9 +226,16 @@ export function useDegradedFields() { isDegradedFieldsValueLoading, isAnalysisInProgress, degradedFieldAnalysis, - degradedFieldAnalysisResult, + degradedFieldAnalysisFormattedResult, toggleCurrentQualityIssues, showCurrentQualityIssues, expandedRenderedItem, + updateNewFieldLimit, + isMitigationInProgress, + isRolloverInProgress, + newFieldLimitData, + isRolloverRequired, + isMitigationAppliedSuccessfully, + triggerRollover, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 9175d06e105b4..827cd4b0a1e49 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,6 +8,8 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { + DataStreamRolloverResponse, + dataStreamRolloverResponseRt, DegradedFieldAnalysis, degradedFieldAnalysisRt, DegradedFieldValues, @@ -19,6 +21,8 @@ import { IntegrationDashboardsResponse, integrationDashboardsRT, IntegrationResponse, + UpdateFieldLimitResponse, + updateFieldLimitResponseRt, } from '../../../common/api_types'; import { DataStreamDetails, @@ -37,6 +41,7 @@ import { Integration } from '../../../common/data_streams_stats/integration'; import { AnalyzeDegradedFieldsParams, GetDataStreamIntegrationParams, + UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; import { DatasetQualityError } from '../../../common/errors'; @@ -196,4 +201,46 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { new DatasetQualityError(`Failed to decode the analysis response: ${message}`) )(response); } + + public async setNewFieldLimit({ + dataStream, + newFieldLimit, + }: UpdateFieldLimitParams): Promise { + const response = await this.http + .put( + `/internal/dataset_quality/data_streams/${dataStream}/update_field_limit`, + { body: JSON.stringify({ newFieldLimit }) } + ) + .catch((error) => { + throw new DatasetQualityError(`Failed to set new Limit: ${error.message}`, error); + }); + + const decodedResponse = decodeOrThrow( + updateFieldLimitResponseRt, + (message: string) => + new DatasetQualityError(`Failed to decode setting of new limit response: ${message}"`) + )(response); + + return decodedResponse; + } + + public async rolloverDataStream({ + dataStream, + }: { + dataStream: string; + }): Promise { + const response = await this.http + .post( + `/internal/dataset_quality/data_streams/${dataStream}/rollover` + ) + .catch((error) => { + throw new DatasetQualityError(`Failed to rollover datastream": ${error}`, error); + }); + + return decodeOrThrow( + dataStreamRolloverResponseRt, + (message: string) => + new DatasetQualityError(`Failed to decode rollover response: ${message}"`) + )(response); + } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index a2f7db99e5af1..6eac8bd732840 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -20,8 +20,15 @@ import { import { AnalyzeDegradedFieldsParams, GetDataStreamIntegrationParams, + UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; -import { Dashboard, DegradedFieldAnalysis, DegradedFieldValues } from '../../../common/api_types'; +import { + Dashboard, + DataStreamRolloverResponse, + DegradedFieldAnalysis, + DegradedFieldValues, + UpdateFieldLimitResponse, +} from '../../../common/api_types'; export type DataStreamDetailsServiceSetup = void; @@ -47,4 +54,6 @@ export interface IDataStreamDetailsClient { params: GetDataStreamIntegrationParams ): Promise; analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise; + setNewFieldLimit(params: UpdateFieldLimitParams): Promise; + rolloverDataStream(params: { dataStream: string }): Promise; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts index b501fd02bdcf3..f5fdd063492a3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts @@ -59,3 +59,28 @@ export const fetchDataStreamIntegrationFailedNotifier = ( text: error.message, }); }; + +export const updateFieldLimitFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.details.updateFieldLimitFailed', { + defaultMessage: "We couldn't update the field limit.", + }), + text: error.message, + }); +}; + +export const rolloverDataStreamFailedNotifier = ( + toasts: IToasts, + error: Error, + dataStream: string +) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.details.rolloverDataStreamFailed', { + defaultMessage: "We couldn't rollover the data stream: {dataStream}.", + values: { + dataStream, + }, + }), + text: error.message, + }); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 352aff140c275..8ac65a7dca4a7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -25,6 +25,7 @@ import { DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, + UpdateFieldLimitResponse, } from '../../../common/api_types'; import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications'; import { @@ -33,6 +34,8 @@ import { fetchDataStreamSettingsFailedNotifier, fetchDataStreamIntegrationFailedNotifier, fetchIntegrationDashboardsFailedNotifier, + updateFieldLimitFailedNotifier, + rolloverDataStreamFailedNotifier, } from './notifications'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -189,10 +192,6 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, done: { on: { - UPDATE_TIME_RANGE: { - target: 'fetching', - actions: ['resetDegradedFieldPageAndRowsPerPage'], - }, UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { target: 'done', actions: ['storeDegradedFieldTableOptions'], @@ -200,7 +199,10 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( OPEN_DEGRADED_FIELD_FLYOUT: { target: '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', - actions: ['storeExpandedDegradedField'], + actions: [ + 'storeExpandedDegradedField', + 'resetFieldLimitServerResponse', + ], }, TOGGLE_CURRENT_QUALITY_ISSUES: { target: 'fetching', @@ -282,48 +284,105 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, open: { - type: 'parallel', + initial: 'initialized', states: { - ignoredValues: { - initial: 'fetching', + initialized: { + type: 'parallel', states: { - fetching: { - invoke: { - src: 'loadDegradedFieldValues', - onDone: { - target: 'done', - actions: ['storeDegradedFieldValues'], - }, - onError: [ - { - target: '#DatasetQualityDetailsController.indexNotFound', - cond: 'isIndexNotFoundError', - }, - { - target: 'done', + ignoredValues: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDegradedFieldValues', + onDone: { + target: 'done', + actions: ['storeDegradedFieldValues'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], }, - ], + }, + done: {}, }, }, - done: {}, - }, - }, - analyze: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'analyzeDegradedField', - onDone: { - target: 'done', - actions: ['storeDegradedFieldAnalysis'], + mitigation: { + initial: 'analyzing', + states: { + analyzing: { + invoke: { + src: 'analyzeDegradedField', + onDone: { + target: 'analyzed', + actions: ['storeDegradedFieldAnalysis'], + }, + onError: { + target: 'analyzed', + }, + }, }, - onError: { - target: 'done', + analyzed: { + on: { + SET_NEW_FIELD_LIMIT: { + target: 'mitigating', + actions: 'storeNewFieldLimit', + }, + }, + }, + mitigating: { + invoke: { + src: 'saveNewFieldLimit', + onDone: [ + { + target: 'askingForRollover', + actions: 'storeNewFieldLimitResponse', + cond: 'hasFailedToUpdateLastBackingIndex', + }, + { + target: 'success', + actions: 'storeNewFieldLimitResponse', + }, + ], + onError: { + target: 'error', + actions: [ + 'storeNewFieldLimitErrorResponse', + 'notifySaveNewFieldLimitError', + ], + }, + }, + }, + askingForRollover: { + on: { + ROLLOVER_DATA_STREAM: { + target: 'rollingOver', + }, + }, + }, + rollingOver: { + invoke: { + src: 'rolloverDataStream', + onDone: { + target: 'success', + actions: ['raiseForceTimeRangeRefresh'], + }, + onError: { + target: 'error', + actions: 'notifySaveNewFieldLimitError', + }, + }, }, + success: {}, + error: {}, }, }, - done: {}, }, }, }, @@ -482,9 +541,28 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( isIndexNotFoundError: true, }; }), + storeNewFieldLimit: assign((_, event) => { + return 'newFieldLimit' in event + ? { fieldLimit: { newFieldLimit: event.newFieldLimit } } + : {}; + }), + storeNewFieldLimitResponse: assign( + (context, event: DoneInvokeEvent) => { + return 'data' in event + ? { fieldLimit: { ...context.fieldLimit, result: event.data, error: false } } + : {}; + } + ), + storeNewFieldLimitErrorResponse: assign((context) => { + return { fieldLimit: { ...context.fieldLimit, error: true } }; + }), + resetFieldLimitServerResponse: assign(() => ({ + fieldLimit: undefined, + })), + raiseForceTimeRangeRefresh: raise('UPDATE_TIME_RANGE'), }, guards: { - checkIfActionForbidden: (context, event) => { + checkIfActionForbidden: (_, event) => { return ( 'data' in event && typeof event.data === 'object' && @@ -516,6 +594,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( hasNoDegradedFieldsSelected: (context) => { return !Boolean(context.expandedDegradedField); }, + hasFailedToUpdateLastBackingIndex: (_, event) => { + return ( + 'data' in event && + typeof event.data === 'object' && + 'isLatestBackingIndexUpdated' in event.data && + !event.data.isLatestBackingIndexUpdated + ); + }, }, } ); @@ -552,6 +638,10 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ 'dataStreamSettings' in context ? context.dataStreamSettings?.integration : undefined; return fetchDataStreamIntegrationFailedNotifier(toasts, event.data, integrationName); }, + notifySaveNewFieldLimitError: (_context, event: DoneInvokeEvent) => + updateFieldLimitFailedNotifier(toasts, event.data), + notifyRolloverDataStreamError: (context, event: DoneInvokeEvent) => + rolloverDataStreamFailedNotifier(toasts, event.data, context.dataStream), }, services: { checkDatasetIsAggregatable: (context) => { @@ -603,7 +693,8 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ dataStream: context.showCurrentQualityIssues && 'dataStreamSettings' in context && - context.dataStreamSettings + context.dataStreamSettings && + context.dataStreamSettings.lastBackingIndexName ? context.dataStreamSettings.lastBackingIndexName : context.dataStream, start, @@ -661,6 +752,21 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ return Promise.resolve(); }, + saveNewFieldLimit: (context) => { + if ('fieldLimit' in context && context.fieldLimit && context.fieldLimit.newFieldLimit) { + return dataStreamDetailsClient.setNewFieldLimit({ + dataStream: context.dataStream, + newFieldLimit: context.fieldLimit.newFieldLimit, + }); + } + + return Promise.resolve(); + }, + rolloverDataStream: (context) => { + return dataStreamDetailsClient.rolloverDataStream({ + dataStream: context.dataStream, + }); + }, }, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index cdf3bfa579e55..cdebcfbe53d86 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -10,12 +10,14 @@ import type { DegradedFieldSortField } from '../../hooks'; import { Dashboard, DataStreamDetails, + DataStreamRolloverResponse, DataStreamSettings, DegradedField, DegradedFieldAnalysis, DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, + UpdateFieldLimitResponse, } from '../../../common/api_types'; import { TableCriteria, TimeRangeConfig } from '../../../common/types'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -37,6 +39,12 @@ export interface DegradedFieldsWithData { data: DegradedField[]; } +export interface FieldLimit { + newFieldLimit?: number; + result?: UpdateFieldLimitResponse; + error?: boolean; +} + export interface WithDefaultControllerState { dataStream: string; degradedFields: DegradedFieldsTableConfig; @@ -48,6 +56,7 @@ export interface WithDefaultControllerState { integration?: Integration; expandedDegradedField?: string; isNonAggregatable?: boolean; + fieldLimit?: FieldLimit; } export interface WithDataStreamDetails { @@ -87,6 +96,16 @@ export interface WithDegradeFieldAnalysis { degradedFieldAnalysis: DegradedFieldAnalysis; } +export interface WithNewFieldLimit { + fieldLimit?: FieldLimit & { + newFieldLimit: number; + }; +} + +export interface WithNewFieldLimitResponse { + fieldLimit: FieldLimit; +} + export type DefaultDatasetQualityDetailsContext = Pick< WithDefaultControllerState, 'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues' @@ -128,38 +147,50 @@ export type DatasetQualityDetailsControllerTypeState = } | { value: - | 'initializing.degradedFieldFlyout.open.ignoredValues.fetching' - | 'initializing.degradedFieldFlyout.open.analyze.fetching'; + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'; + context: WithDefaultControllerState & WithDataStreamSettings; + } + | { + value: + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'; + context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration; + } + | { + value: 'initializing.degradedFieldFlyout.open'; + context: WithDefaultControllerState; + } + | { + value: + | 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing'; context: WithDefaultControllerState & WithDegradedFieldsData; } | { - value: 'initializing.degradedFieldFlyout.open.ignoredValues.done'; + value: 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.done'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues; } | { - value: 'initializing.degradedFieldFlyout.open.analyze.done'; + value: + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.success' + | 'initializing.degradedFieldFlyout.open.initialized.mitigation.error'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradeFieldAnalysis; } | { - value: 'initializing.degradedFieldFlyout.open'; + value: 'initializing.degradedFieldFlyout.open.initialized.mitigation.success'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues & - WithDegradeFieldAnalysis; - } - | { - value: - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'; - context: WithDefaultControllerState & WithDataStreamSettings; - } - | { - value: - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'; - context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration; + WithDegradeFieldAnalysis & + WithNewFieldLimit & + WithNewFieldLimitResponse; }; export type DatasetQualityDetailsControllerContext = @@ -188,6 +219,13 @@ export type DatasetQualityDetailsControllerEvent = type: 'UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA'; degraded_field_criteria: TableCriteria; } + | { + type: 'SET_NEW_FIELD_LIMIT'; + newFieldLimit: number; + } + | { + type: 'ROLLOVER_DATA_STREAM'; + } | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent @@ -197,4 +235,6 @@ export type DatasetQualityDetailsControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent - | DoneInvokeEvent; + | DoneInvokeEvent + | DoneInvokeEvent + | DoneInvokeEvent; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index eb1d70b867dc4..288eff11b92a8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -29,8 +29,6 @@ export async function getDataStreamSettings({ esClient: ElasticsearchClient; dataStream: string; }): Promise { - throwIfInvalidDataStreamParams(dataStream); - const [createdOn, [dataStreamInfo], datasetUserPrivileges] = await Promise.all([ getDataStreamCreatedOn(esClient, dataStream), dataStreamService.getMatchingDataStreams(esClient, dataStream), @@ -39,12 +37,14 @@ export async function getDataStreamSettings({ const integration = dataStreamInfo?._meta?.package?.name; const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0]; + const indexTemplate = dataStreamInfo?.template; return { createdOn, integration, datasetUserPrivileges, lastBackingIndexName: lastBackingIndex?.index_name, + indexTemplate, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts index 433086c0b3e52..cbaa637dc60bc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts @@ -13,6 +13,7 @@ export interface DataStreamSettingResponse { totalFieldLimit: number; ignoreDynamicBeyondLimit?: boolean; ignoreMalformed?: boolean; + defaultPipeline?: string; } const DEFAULT_FIELD_LIMIT = 1000; @@ -28,16 +29,20 @@ export async function getDataStreamSettings({ lastBackingIndex: string; }): Promise { const settings = await datasetQualityESClient.settings({ index: dataStream }); - const indexSettings = settings[lastBackingIndex]?.settings?.index?.mapping; + const setting = settings[lastBackingIndex]?.settings; + const mappingsInsideSettings = setting?.index?.mapping; return { - nestedFieldLimit: indexSettings?.nested_fields?.limit - ? Number(indexSettings?.nested_fields?.limit) + nestedFieldLimit: mappingsInsideSettings?.nested_fields?.limit + ? Number(mappingsInsideSettings?.nested_fields?.limit) : DEFAULT_NESTED_FIELD_LIMIT, - totalFieldLimit: indexSettings?.total_fields?.limit - ? Number(indexSettings?.total_fields?.limit) + totalFieldLimit: mappingsInsideSettings?.total_fields?.limit + ? Number(mappingsInsideSettings?.total_fields?.limit) : DEFAULT_FIELD_LIMIT, - ignoreDynamicBeyondLimit: toBoolean(indexSettings?.total_fields?.ignore_dynamic_beyond_limit), - ignoreMalformed: toBoolean(indexSettings?.ignore_malformed), + ignoreDynamicBeyondLimit: toBoolean( + mappingsInsideSettings?.total_fields?.ignore_dynamic_beyond_limit + ), + ignoreMalformed: toBoolean(mappingsInsideSettings?.ignore_malformed), + defaultPipeline: setting?.index?.default_pipeline, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts index a0e7606b475b2..97ff0b124aae9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts @@ -28,7 +28,13 @@ export async function analyzeDegradedField({ const [ { fieldCount, fieldPresent, fieldMapping }, - { nestedFieldLimit, totalFieldLimit, ignoreDynamicBeyondLimit, ignoreMalformed }, + { + nestedFieldLimit, + totalFieldLimit, + ignoreDynamicBeyondLimit, + ignoreMalformed, + defaultPipeline, + }, ] = await Promise.all([ getDataStreamMapping({ datasetQualityESClient, @@ -48,5 +54,6 @@ export async function analyzeDegradedField({ totalFieldLimit, ignoreMalformed, nestedFieldLimit, + defaultPipeline, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 047004d58a6a2..41ba3ee8c7299 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -16,6 +16,8 @@ import { DatasetUserPrivileges, DegradedFieldValues, DegradedFieldAnalysis, + UpdateFieldLimitResponse, + DataStreamRolloverResponse, } from '../../../common/api_types'; import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; @@ -29,6 +31,8 @@ import { getDegradedFields } from './get_degraded_fields'; import { getDegradedFieldValues } from './get_degraded_field_values'; import { analyzeDegradedField } from './get_degraded_field_analysis'; import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; +import { updateFieldLimit } from './update_field_limit'; +import { createDatasetQualityESClient } from '../../utils'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -324,6 +328,58 @@ const analyzeDegradedFieldRoute = createDatasetQualityServerRoute({ }, }); +const updateFieldLimitRoute = createDatasetQualityServerRoute({ + endpoint: 'PUT /internal/dataset_quality/data_streams/{dataStream}/update_field_limit', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + body: t.type({ + newFieldLimit: t.number, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const updatedLimitResponse = await updateFieldLimit({ + esClient, + newFieldLimit: params.body.newFieldLimit, + dataStream: params.path.dataStream, + }); + + return updatedLimitResponse; + }, +}); + +const rolloverDataStream = createDatasetQualityServerRoute({ + endpoint: 'POST /internal/dataset_quality/data_streams/{dataStream}/rollover', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const { acknowledged } = await datasetQualityESClient.rollover({ + alias: params.path.dataStream, + }); + + return { acknowledged }; + }, +}); + export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, @@ -334,4 +390,6 @@ export const dataStreamsRouteRepository = { ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, ...analyzeDegradedFieldRoute, + ...updateFieldLimitRoute, + ...rolloverDataStream, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts new file mode 100644 index 0000000000000..f377ea1e2642c --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { badRequest } from '@hapi/boom'; +import { createDatasetQualityESClient } from '../../../utils'; +import { updateComponentTemplate } from './update_component_template'; +import { updateLastBackingIndexSettings } from './update_settings_last_backing_index'; +import { UpdateFieldLimitResponse } from '../../../../common/api_types'; +import { getDataStreamSettings } from '../get_data_stream_details'; + +export async function updateFieldLimit({ + esClient, + newFieldLimit, + dataStream, +}: { + esClient: ElasticsearchClient; + newFieldLimit: number; + dataStream: string; +}): Promise { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const { lastBackingIndexName, indexTemplate } = await getDataStreamSettings({ + esClient, + dataStream, + }); + + if (!lastBackingIndexName || !indexTemplate) { + throw badRequest(`Data stream does not exists. Received value "${dataStream}"`); + } + + const { + acknowledged: isComponentTemplateUpdated, + componentTemplateName, + error: errorUpdatingComponentTemplate, + } = await updateComponentTemplate({ datasetQualityESClient, indexTemplate, newFieldLimit }); + + if (errorUpdatingComponentTemplate) { + throw badRequest(errorUpdatingComponentTemplate); + } + + const { acknowledged: isLatestBackingIndexUpdated, error: errorUpdatingBackingIndex } = + await updateLastBackingIndexSettings({ + datasetQualityESClient, + lastBackingIndex: lastBackingIndexName, + newFieldLimit, + }); + + return { + isComponentTemplateUpdated, + isLatestBackingIndexUpdated, + customComponentTemplateName: componentTemplateName, + error: errorUpdatingBackingIndex, + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts new file mode 100644 index 0000000000000..0bf19410bd6ac --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../common/utils/component_template_name'; + +interface UpdateComponentTemplateResponse { + acknowledged: boolean | undefined; + componentTemplateName: string; + error?: string; +} + +export async function updateComponentTemplate({ + datasetQualityESClient, + indexTemplate, + newFieldLimit, +}: { + datasetQualityESClient: DatasetQualityESClient; + indexTemplate: string; + newFieldLimit: number; +}): Promise { + const newSettings = { + settings: { + 'index.mapping.total_fields.limit': newFieldLimit, + }, + }; + + const customComponentTemplateName = `${getComponentTemplatePrefixFromIndexTemplate( + indexTemplate + )}@custom`; + + try { + const { acknowledged } = await datasetQualityESClient.updateComponentTemplate({ + name: customComponentTemplateName, + template: newSettings, + }); + + return { + acknowledged, + componentTemplateName: customComponentTemplateName, + }; + } catch (error) { + return { + acknowledged: undefined, // acknowledge is undefined when the request fails + componentTemplateName: customComponentTemplateName, + error: error.message, + }; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts new file mode 100644 index 0000000000000..b98a315547554 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; + +interface UpdateLastBackingIndexSettingsResponse { + acknowledged: boolean | undefined; + error?: string; +} + +export async function updateLastBackingIndexSettings({ + datasetQualityESClient, + lastBackingIndex, + newFieldLimit, +}: { + datasetQualityESClient: DatasetQualityESClient; + lastBackingIndex: string; + newFieldLimit: number; +}): Promise { + const newSettings = { + 'index.mapping.total_fields.limit': newFieldLimit, + }; + + try { + const { acknowledged } = await datasetQualityESClient.updateSettings({ + index: lastBackingIndex, + settings: newSettings, + }); + + return { acknowledged }; + } catch (error) { + return { + acknowledged: undefined, // acknowledge is undefined when the request fails + error: error.message, + }; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index baa2403690fd8..8a78b4163da95 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -8,11 +8,16 @@ import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { ElasticsearchClient } from '@kbn/core/server'; import { + ClusterPutComponentTemplateRequest, + ClusterPutComponentTemplateResponse, FieldCapsRequest, FieldCapsResponse, Indices, IndicesGetMappingResponse, IndicesGetSettingsResponse, + IndicesPutSettingsRequest, + IndicesPutSettingsResponse, + IndicesRolloverResponse, } from '@elastic/elasticsearch/lib/api/types'; type DatasetQualityESSearchParams = ESSearchRequest & { @@ -23,12 +28,12 @@ export type DatasetQualityESClient = ReturnType( + search( searchParams: TParams ): Promise> { return esClient.search(searchParams) as Promise; }, - async msearch( + msearch( index = {} as { index?: Indices }, searches: TParams[] ): Promise<{ @@ -38,14 +43,25 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) { searches: searches.map((search) => [index, search]).flat(), }) as Promise; }, - async fieldCaps(params: FieldCapsRequest): Promise { - return esClient.fieldCaps(params) as Promise; + fieldCaps(params: FieldCapsRequest): Promise { + return esClient.fieldCaps(params); }, - async mappings(params: { index: string }): Promise { + mappings(params: { index: string }): Promise { return esClient.indices.getMapping(params); }, - async settings(params: { index: string }): Promise { + settings(params: { index: string }): Promise { return esClient.indices.getSettings(params); }, + updateComponentTemplate( + params: ClusterPutComponentTemplateRequest + ): Promise { + return esClient.cluster.putComponentTemplate(params); + }, + updateSettings(params: IndicesPutSettingsRequest): Promise { + return esClient.indices.putSettings(params); + }, + rollover(params: { alias: string }): Promise { + return esClient.indices.rollover(params); + }, }; } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts new file mode 100644 index 0000000000000..d11d93d9eae51 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const start = '2024-10-17T11:00:00.000Z'; + const end = '2024-10-17T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials.post( + `/internal/dataset_quality/data_streams/${dataStream}/rollover` + ); + } + + describe('Datastream Rollover', function () { + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); + + it('returns acknowledged when rollover is successful', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + expect(resp.body.acknowledged).to.be(true); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts new file mode 100644 index 0000000000000..16c463e8caf6c --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { + createBackingIndexNameWithoutVersion, + getDataStreamSettingsOfEarliestIndex, + rolloverDataStream, +} from './utils/es_utils'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const esClient = getService('es'); + const packageApi = getService('packageApi'); + const config = getService('config'); + const isServerless = !!config.get('serverless'); + const start = '2024-09-20T11:00:00.000Z'; + const end = '2024-09-20T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const syntheticsDataset = 'synthetics'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + const syntheticsDataStreamName = `${type}-${syntheticsDataset}-${namespace}`; + + const defaultDataStreamPrivileges = { + datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, + }; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials.get( + `/internal/dataset_quality/data_streams/${dataStream}/settings` + ); + } + + describe('Dataset quality settings', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + }); + + after(async () => { + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('returns only privileges if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: nonExistentDataStream, + }, + }); + expect(resp.body).eql(defaultDataStreamPrivileges); + }); + + describe('gets the data stream settings for non integrations', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + after(async () => { + await synthtrace.clean(); + }); + + it('returns "createdOn", "indexTemplate" and "lastBackingIndexName" correctly when available for non integration', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + dataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.indexTemplate).to.be('logs'); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace, + })}-000001` + ); + expect(resp.body.datasetUserPrivileges).to.eql( + defaultDataStreamPrivileges.datasetUserPrivileges + ); + }); + + it('returns "createdOn", "indexTemplate" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, dataStreamName); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + dataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` + ); + expect(resp.body.indexTemplate).to.be('logs'); + }); + }); + + describe('gets the data stream settings for integrations', () => { + before(async () => { + await packageApi.installPackage({ + roleAuthc: adminRoleAuthc, + pkg: syntheticsDataset, + }); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(syntheticsDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + after(async () => { + await synthtrace.clean(); + await packageApi.uninstallPackage({ + roleAuthc: adminRoleAuthc, + pkg: syntheticsDataset, + }); + }); + + it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly when available for integration', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + syntheticsDataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: syntheticsDataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.indexTemplate).to.be('logs'); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: syntheticsDataset, + namespace, + })}-000001` + ); + expect(resp.body.datasetUserPrivileges).to.eql( + defaultDataStreamPrivileges.datasetUserPrivileges + ); + }); + + it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, syntheticsDataStreamName); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + syntheticsDataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: syntheticsDataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: syntheticsDataset, + namespace, + })}-000002` + ); + expect(resp.body.indexTemplate).to.be('logs'); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts index ed06b81d647fb..e9e2665548764 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts @@ -9,7 +9,7 @@ import expect from '@kbn/expect'; import { log, timerange } from '@kbn/apm-synthtrace-client'; import { SupertestWithRoleScopeType } from '../../../services'; import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; -import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './es_utils'; +import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './utils/es_utils'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; const MORE_THAN_1024_CHARS = diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 0c660dda0a445..7e555b7a310e1 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -11,5 +11,8 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) describe('Dataset quality', () => { loadTestFile(require.resolve('./integrations')); loadTestFile(require.resolve('./degraded_field_analyze')); + loadTestFile(require.resolve('./data_stream_settings')); + loadTestFile(require.resolve('./data_stream_rollover')); + loadTestFile(require.resolve('./update_field_limit')); }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts index 910dd84bb309e..33b3fccbea8a1 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts @@ -5,15 +5,15 @@ * 2.0. */ -import { RoleCredentials, InternalRequestHeader } from '@kbn/ftr-common-functional-services'; import expect from '@kbn/expect'; import { APIReturnType } from '@kbn/dataset-quality-plugin/common/rest'; import { CustomIntegration } from '../../../services/package_api'; import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { const samlAuth = getService('samlAuth'); - const supertestWithoutAuth = getService('supertestWithoutAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); const packageApi = getService('packageApi'); const endpoint = 'GET /internal/dataset_quality/integrations'; @@ -33,27 +33,30 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { ]; async function callApiAs({ - roleAuthc, - headers, + roleScopedSupertestWithCookieCredentials, }: { - roleAuthc: RoleCredentials; - headers: InternalRequestHeader; + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; }): Promise { - const { body } = await supertestWithoutAuth - .get('/internal/dataset_quality/integrations') - .set(roleAuthc.apiKeyHeader) - .set(headers); + const { body } = await roleScopedSupertestWithCookieCredentials.get( + '/internal/dataset_quality/integrations' + ); return body; } describe('Integrations', () => { let adminRoleAuthc: RoleCredentials; - let internalHeaders: InternalRequestHeader; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; before(async () => { adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); - internalHeaders = samlAuth.getInternalRequestHeader(); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); }); after(async () => { @@ -74,8 +77,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { it('returns all installed integrations and its datasets map', async () => { const body = await callApiAs({ - roleAuthc: adminRoleAuthc, - headers: internalHeaders, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, }); expect(body.integrations.map((integration: Integration) => integration.name)).to.eql([ @@ -108,8 +110,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { it('returns custom integrations and its datasets map', async () => { const body = await callApiAs({ - roleAuthc: adminRoleAuthc, - headers: internalHeaders, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, }); expect(body.integrations.map((integration: Integration) => integration.name)).to.eql([ diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts new file mode 100644 index 0000000000000..3f842cd43f3e2 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { log, timerange } from '@kbn/apm-synthtrace-client'; + +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; +import { createBackingIndexNameWithoutVersion, rolloverDataStream } from './utils/es_utils'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const esClient = getService('es'); + const packageApi = getService('packageApi'); + const start = '2024-10-17T11:00:00.000Z'; + const end = '2024-10-17T11:01:00.000Z'; + const type = 'logs'; + const invalidDataset = 'invalid'; + const integrationsDataset = 'nginx.access'; + const pkg = 'nginx'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const invalidDataStreamName = `${type}-${invalidDataset}-${namespace}`; + const integrationsDataStreamName = `${type}-${integrationsDataset}-${namespace}`; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream, fieldLimit }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + fieldLimit: number; + }; + }) { + return roleScopedSupertestWithCookieCredentials + .put(`/internal/dataset_quality/data_streams/${dataStream}/update_field_limit`) + .send({ + newFieldLimit: fieldLimit, + }); + } + + describe('Update field limit', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + await packageApi.installPackage({ + roleAuthc: adminRoleAuthc, + pkg, + }); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(integrationsDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + after(async () => { + await synthtrace.clean(); + await packageApi.uninstallPackage({ + roleAuthc: adminRoleAuthc, + pkg, + }); + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('should handles failure gracefully when invalid datastream provided ', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: invalidDataStreamName, + fieldLimit: 10, + }, + }); + + expect(resp.body.statusCode).to.be(400); + expect(resp.body.message).to.be( + `Data stream does not exists. Received value "${invalidDataStreamName}"` + ); + }); + + it('should update last backing index and custom component template', async () => { + // We rollover the data stream to create a new backing index + await rolloverDataStream(esClient, integrationsDataStreamName); + + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: integrationsDataStreamName, + fieldLimit: 50, + }, + }); + + expect(resp.body.isComponentTemplateUpdated).to.be(true); + expect(resp.body.isLatestBackingIndexUpdated).to.be(true); + expect(resp.body.customComponentTemplateName).to.be(`${type}-${integrationsDataset}@custom`); + expect(resp.body.error).to.be(undefined); + + const { component_templates: componentTemplates } = + await esClient.cluster.getComponentTemplate({ + name: `${type}-${integrationsDataset}@custom`, + }); + + const customTemplate = componentTemplates.filter( + (tmp) => tmp.name === `${type}-${integrationsDataset}@custom` + ); + + expect(customTemplate).to.have.length(1); + expect( + customTemplate[0].component_template.template.settings?.index?.mapping?.total_fields?.limit + ).to.be('50'); + + const settingsForAllIndices = await esClient.indices.getSettings({ + index: integrationsDataStreamName, + }); + + const backingIndexWithoutVersion = createBackingIndexNameWithoutVersion({ + type, + dataset: integrationsDataset, + namespace, + }); + const settingsForLastBackingIndex = + settingsForAllIndices[backingIndexWithoutVersion + '-000002'].settings; + const settingsForPreviousBackingIndex = + settingsForAllIndices[backingIndexWithoutVersion + '-000001'].settings; + + // Only the Last Backing Index should have the updated limit and not the one previous to it + expect(settingsForLastBackingIndex?.index?.mapping?.total_fields?.limit).to.be('50'); + + // The previous one should have the default limit of 1000 + expect(settingsForPreviousBackingIndex?.index?.mapping?.total_fields?.limit).to.be('1000'); + + // Rollover to test custom component template + await rolloverDataStream(esClient, integrationsDataStreamName); + + const settingsForLatestBackingIndex = await esClient.indices.getSettings({ + index: backingIndexWithoutVersion + '-000003', + }); + + // The new backing index should read settings from custom component template + expect( + settingsForLatestBackingIndex[backingIndexWithoutVersion + '-000003'].settings?.index + ?.mapping?.total_fields?.limit + ).to.be('50'); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts similarity index 63% rename from x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts index 0e041781122cd..a2fa712ba3be7 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts @@ -39,3 +39,21 @@ export async function setDataStreamSettings( settings, }); } + +export async function rolloverDataStream(es: Client, name: string) { + return es.indices.rollover({ alias: name }); +} + +export async function getDataStreamSettingsOfEarliestIndex(es: Client, name: string) { + const matchingIndexesObj = await es.indices.getSettings({ index: name }); + + const matchingIndexes = Object.keys(matchingIndexesObj ?? {}); + matchingIndexes.sort((a, b) => { + return ( + Number(matchingIndexesObj[a].settings?.index?.creation_date) - + Number(matchingIndexesObj[b].settings?.index?.creation_date) + ); + }); + + return matchingIndexesObj[matchingIndexes[0]].settings; +} diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts deleted file mode 100644 index 45f37b44983aa..0000000000000 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { log, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; -import { DatasetQualityApiClientKey } from '../../common/config'; -import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { - expectToReject, - getDataStreamSettingsOfEarliestIndex, - rolloverDataStream, -} from '../../utils'; -import { createBackingIndexNameWithoutVersion } from './es_utils'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const synthtrace = getService('logSynthtraceEsClient'); - const esClient = getService('es'); - const datasetQualityApiClient = getService('datasetQualityApiClient'); - const pkgService = getService('packageService'); - const start = '2023-12-11T18:00:00.000Z'; - const end = '2023-12-11T18:01:00.000Z'; - const type = 'logs'; - const dataset = 'synth.1'; - const integrationDataset = 'apache.access'; - const namespace = 'default'; - const serviceName = 'my-service'; - const hostName = 'synth-host'; - const pkg = { - name: 'apache', - version: '1.14.0', - }; - - const defaultDataStreamPrivileges = { - datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, - }; - - async function callApiAs(user: DatasetQualityApiClientKey, dataStream: string) { - return await datasetQualityApiClient[user]({ - endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', - params: { - path: { - dataStream, - }, - }, - }); - } - - registry.when('DataStream Settings', { config: 'basic' }, () => { - describe('gets the data stream settings', () => { - before(async () => { - // Install Integration and ingest logs for it - await pkgService.installPackage(pkg); - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(integrationDataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - // Ingest basic logs - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(dataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - }); - - it('returns error when dataStream param is not provided', async () => { - const expectedMessage = 'Data Stream name cannot be empty'; - const err = await expectToReject(() => - callApiAs('datasetQualityMonitorUser', encodeURIComponent(' ')) - ); - expect(err.res.status).to.be(400); - expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); - }); - - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApiAs('datasetQualityMonitorUser', nonExistentDataStream); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - - it('returns "createdOn", "integration" and "lastBackingIndexName" correctly when available', async () => { - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${integrationDataset}-${namespace}` - ); - const resp = await callApiAs( - 'datasetQualityMonitorUser', - `${type}-${integrationDataset}-${namespace}` - ); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.integration).to.be('apache'); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ - type, - dataset: integrationDataset, - namespace, - })}-000001` - ); - expect(resp.body.datasetUserPrivileges).to.eql( - defaultDataStreamPrivileges.datasetUserPrivileges - ); - }); - - it('returns "createdOn" and "lastBackingIndexName" for rolled over dataStream', async () => { - await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApiAs( - 'datasetQualityMonitorUser', - `${type}-${dataset}-${namespace}` - ); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` - ); - }); - - after(async () => { - await synthtrace.clean(); - await pkgService.uninstallPackage(pkg); - }); - }); - }); -} diff --git a/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts b/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts new file mode 100644 index 0000000000000..210d5fd349880 --- /dev/null +++ b/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; + +export const logsNginxMappings = (dataset: string): MappingTypeMapping => ({ + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + cloud: { + properties: { + image: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + value: dataset, + }, + namespace: { + type: 'constant_keyword', + value: 'default', + }, + type: { + type: 'constant_keyword', + value: 'logs', + }, + }, + }, + ecs: { + properties: { + version: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + error: { + properties: { + message: { + type: 'match_only_text', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + module: { + type: 'constant_keyword', + value: 'nginx', + }, + }, + }, + host: { + properties: { + containerized: { + type: 'boolean', + }, + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + os: { + properties: { + build: { + type: 'keyword', + ignore_above: 1024, + }, + codename: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + input: { + properties: { + type: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + log: { + properties: { + level: { + type: 'keyword', + ignore_above: 1024, + }, + offset: { + type: 'long', + }, + }, + }, + network: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + nginx: { + properties: { + access: { + properties: { + remote_ip_list: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + }, + }, + test_field: { + type: 'keyword', + ignore_above: 1024, + }, + tls: { + properties: { + established: { + type: 'boolean', + }, + }, + }, + trace: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, +}); diff --git a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts index 2506201aa3b85..1b477644feca0 100644 --- a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts @@ -17,6 +17,7 @@ import { MORE_THAN_1024_CHARS, } from './data'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; +import { logsNginxMappings } from './custom_mappings/custom_integration_mappings'; export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { const PageObjects = getPageObjects([ @@ -30,34 +31,42 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const esClient = getService('es'); const retry = getService('retry'); const to = new Date().toISOString(); + const type = 'logs'; const degradedDatasetName = 'synth.degraded'; - const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const degradedDataStreamName = `${type}-${degradedDatasetName}-${defaultNamespace}`; const degradedDatasetWithLimitsName = 'synth.degraded.rca'; - const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`; + const degradedDatasetWithLimitDataStreamName = `${type}-${degradedDatasetWithLimitsName}-${defaultNamespace}`; const serviceName = 'test_service'; const count = 5; const customComponentTemplateName = 'logs-synth@mappings'; - describe('Degraded fields flyout', () => { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - }); - - after(async () => { - await synthtrace.clean(); - }); + const nginxAccessDatasetName = 'nginx.access'; + const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; + const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; + const nginxPkg = { + name: 'nginx', + version: '1.23.0', + }; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); it('should open and close the flyout when user clicks on the expand button', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, @@ -90,31 +99,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); }); - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - - describe('testing root cause for ignored fields', () => { + describe('detecting root cause for ignored fields', () => { before(async () => { // Create custom component template await synthtrace.createComponentTemplate( @@ -142,8 +127,18 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid hidden: false, }, }); - // Ingest Degraded Logs with 25 fields + + // Install Nginx Integration and ingest logs for it + await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); + + // Create custom component template to avoid issues with LogsDB + await synthtrace.createComponentTemplate( + customComponentTemplateNameNginx, + logsNginxMappings(nginxAccessDatasetName) + ); + await synthtrace.index([ + // Ingest Degraded Logs with 25 fields in degraded DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -161,7 +156,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .defaults({ 'service.name': serviceName, 'trace.id': generateShortId(), - test_field: [MORE_THAN_1024_CHARS, 'hello world'], + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 42 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], }) .timestamp(timestamp) ); @@ -176,8 +194,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid } ); - // Ingest Degraded Logs with 26 field + // Set Limit of 42 + await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { + 'mapping.total_fields.limit': 42, + }); + await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -196,7 +219,31 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -205,9 +252,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: degradedDatasetWithLimitsName, + }) + '-000002', + { + 'mapping.total_fields.limit': 26, + } + ); + + // Set Limit of 43 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: nginxAccessDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 43, + } + ); - // Ingest docs with 26 fields again await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -223,11 +291,34 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .service(serviceName) .namespace(defaultNamespace) .defaults({ - 'log.file.path': '/my-service.log', 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -235,8 +326,128 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ]); }); - describe('field character limit exceeded', () => { - it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(4); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(3); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + describe('character limit exceeded', () => { + it('should display cause as "field character limit exceeded" when a field is ignored due to character limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', @@ -253,25 +464,167 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.closeFlyout(); }); - it('should display values when cause is "field ignored"', async () => { + it('should display values when cause is "field character limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', }); await retry.tryForTime(5000, async () => { - const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + const testFieldValue1Exists = await PageObjects.datasetQuality.doesTextExist( 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', MORE_THAN_1024_CHARS ); - expect(testFieldValueExists).to.be(true); + const testFieldValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + expect(testFieldValue1Exists).to.be(true); + expect(testFieldValue2Exists).to.be(true); }); await PageObjects.datasetQuality.closeFlyout(); }); + + it('should display the maximum character limit when cause is "field character limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const limitValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-characterLimit', + '1024' + ); + expect(limitValueExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should show possible mitigation section with manual options for non integrations', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + // Should point to index template with the datastream name as value + expect(componentTemplateUrl).to.be( + `/data/index_management/templates/${degradedDatasetWithLimitDataStreamName}` + ); + + const nonIntegrationCustomName = `${type}@custom`; + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(nonIntegrationCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + nonIntegrationCustomName + )}` + ); + }); + + it('should show possible mitigation section with different manual options for integrations', async () => { + // Navigate to Integration Dataset + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + const integrationSpecificCustomName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to component template with @custom as value + expect(componentTemplateUrl).to.be( + `/data/index_management/component_templates/${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(integrationSpecificCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + }); }); - describe('field limit exceeded', () => { + describe('past field limit exceeded', () => { it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -289,7 +642,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.closeFlyout(); }); - it('should display the limit when the cause is "field limit exceeded"', async () => { + it('should display the current field limit when the cause is "field limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'cloud', @@ -319,122 +672,194 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); }); - describe('current quality issues', () => { - it('should display issues only from latest backing index when current issues toggle is on', async () => { + describe('current field limit issues', () => { + it('should display increase field limit as a possible mitigation for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); - - expect(currentIssuesToggleState).to.be(false); - - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - - expect(rows.length).to.eql(3); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); - const newCurrentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - expect(newCurrentIssuesToggleState).to.be(true); + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - const newRows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + const linkURL = await linkButton.getAttribute('href'); - expect(newRows.length).to.eql(2); + expect(linkURL).to.be( + 'https://www.elastic.co/guide/en/elasticsearch/reference/master/mapping-settings-limit.html' + ); }); - it('should keep the toggle on when url state says so', async () => { + it('should display increase field limit as a possible mitigation for non integration', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + expandedDegradedField: 'cloud.project', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - expect(currentIssuesToggleState).to.be(true); + // Should not display the panel to increase field limit + await testSubjects.missingOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); + + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); }); - it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + it('should display additional input fields and button increasing the limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Check value in Table - const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const countColumn = table['Docs count']; - expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + // Should display current field limit + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingCurrentLimitFieldText'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const countValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '5' - ); - expect(countValue).to.be(true); - }); + const currentFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingCurrentLimitFieldText' + ); - // Toggle the switch - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const currentFieldLimitValue = await currentFieldLimitInput.getAttribute('value'); + const currentFieldLimit = parseInt(currentFieldLimitValue as string, 10); + const currentFieldLimitDisabledStatus = await currentFieldLimitInput.getAttribute( + 'disabled' ); - // Check value in Table - const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const newCountColumn = newTable['Docs count']; - expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + expect(currentFieldLimit).to.be(43); + expect(currentFieldLimitDisabledStatus).to.be('true'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const newCountValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '15' - ); - expect(newCountValue).to.be(true); - }); + // Should display new field limit + await testSubjects.existOrFail( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitValue = await newFieldLimitInput.getAttribute('value'); + const newFieldLimit = parseInt(newFieldLimitValue as string, 10); + + // Should be 30% more the current limit + const newLimit = Math.round(currentFieldLimit * 1.3); + expect(newFieldLimit).to.be(newLimit); + + // Should display the apply button + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be(null); }); - it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + it('should validate input for new field limit', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + // Should not allow values less than current limit of 43 + await testSubjects.setValue( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText', + '42', + { + clearWithKeyboard: true, + typeCharByChar: true, + } + ); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be('true'); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' ); + const invalidStatus = await newFieldLimitInput.getAttribute('aria-invalid'); + + expect(invalidStatus).to.be('true'); }); - it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + it('should let user increase the field limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + await applyButton.click(); + + await retry.tryForTime(5000, async () => { + // Should display the success callout + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetSuccessCallout' + ); + + // Should display link to component template edited + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + + const ctLink = await testSubjects.find( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + const ctLinkURL = await ctLink.getAttribute('href'); + + const componentTemplateName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to the component template page + expect( + ctLinkURL?.endsWith( + `/data/index_management/component_templates/${encodeURIComponent( + componentTemplateName + )}` + ) + ).to.be(true); + }); + + // Refresh the time range to get the latest data + await PageObjects.datasetQuality.refreshDetailsPageData(); + + // The page should now handle this as ignore_malformed issue and show a warning + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist' ); + // Should not display the panel to increase field limit await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); }); }); @@ -445,6 +870,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid name: degradedDatasetWithLimitDataStreamName, }); await synthtrace.deleteComponentTemplate(customComponentTemplateName); + await PageObjects.observabilityLogsExplorer.uninstallPackage(nginxPkg); + await synthtrace.deleteComponentTemplate(customComponentTemplateNameNginx); }); }); }); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index ccd48e220064a..cef881fe0797c 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -204,6 +204,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv } }, + async waitUntilPossibleMitigationsLoaded() { + await find.waitForDeletedByCssSelector('.euiFlyoutBody .euiSkeletonRectangle', 20 * 1000); + }, + async waitUntilDegradedFieldFlyoutLoaded() { await testSubjects.existOrFail(testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout); }, @@ -239,6 +243,18 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ); }, + generateBackingIndexNameWithoutVersion({ + type = 'logs', + dataset, + namespace = 'default', + }: { + type?: string; + dataset: string; + namespace?: string; + }) { + return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`; + }, + getDatasetsTable(): Promise { return testSubjects.find(testSubjectSelectors.datasetQualityTable); }, @@ -554,3 +570,12 @@ async function getDatasetTableHeaderTexts(tableWrapper: WebElementWrapper) { headerElementWrappers.map((headerElementWrapper) => headerElementWrapper.getVisibleText()) ); } + +function getCurrentDateFormatted() { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}.${month}.${day}`; +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts deleted file mode 100644 index a132bc01c9720..0000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { log, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; -import type { InternalRequestHeader, RoleCredentials } from '../../../../shared/services'; -import { expectToReject, getDataStreamSettingsOfEarliestIndex, rolloverDataStream } from './utils'; -import { - DatasetQualityApiClient, - DatasetQualityApiError, -} from './common/dataset_quality_api_supertest'; -import { DatasetQualityFtrContextProvider } from './common/services'; -import { createBackingIndexNameWithoutVersion } from './utils'; - -export default function ({ getService }: DatasetQualityFtrContextProvider) { - const datasetQualityApiClient: DatasetQualityApiClient = getService('datasetQualityApiClient'); - const synthtrace = getService('logSynthtraceEsClient'); - const svlCommonApi = getService('svlCommonApi'); - const svlUserManager = getService('svlUserManager'); - const esClient = getService('es'); - const start = '2023-12-11T18:00:00.000Z'; - const end = '2023-12-11T18:01:00.000Z'; - const type = 'logs'; - const dataset = 'nginx.access'; - const namespace = 'default'; - const serviceName = 'my-service'; - const hostName = 'synth-host'; - - const defaultDataStreamPrivileges = { - datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, - }; - - async function callApi( - dataStream: string, - roleAuthc: RoleCredentials, - internalReqHeader: InternalRequestHeader - ) { - return await datasetQualityApiClient.slsUser({ - endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', - params: { - path: { - dataStream, - }, - }, - roleAuthc, - internalReqHeader, - }); - } - - describe('gets the data stream settings', () => { - let roleAuthc: RoleCredentials; - let internalReqHeader: InternalRequestHeader; - before(async () => { - roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('admin'); - internalReqHeader = svlCommonApi.getInternalRequestHeader(); - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(dataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - }); - - after(async () => { - await synthtrace.clean(); - await svlUserManager.invalidateM2mApiKeyWithRoleScope(roleAuthc); - }); - - it('returns error when dataStream param is not provided', async () => { - const expectedMessage = 'Data Stream name cannot be empty'; - const err = await expectToReject(() => - callApi(encodeURIComponent(' '), roleAuthc, internalReqHeader) - ); - expect(err.res.status).to.be(400); - expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); - }); - - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApi(nonExistentDataStream, roleAuthc, internalReqHeader); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - - it('returns "createdOn" and "lastBackingIndexName" correctly', async () => { - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ - type, - dataset, - namespace, - })}-000001` - ); - }); - - it('returns "createdOn" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { - await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` - ); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts index 86c8527139846..39b6a3cb476c1 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts @@ -9,7 +9,6 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Dataset Quality', function () { loadTestFile(require.resolve('./data_stream_details')); - loadTestFile(require.resolve('./data_stream_settings')); loadTestFile(require.resolve('./degraded_field_values')); }); } diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts new file mode 100644 index 0000000000000..210d5fd349880 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; + +export const logsNginxMappings = (dataset: string): MappingTypeMapping => ({ + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + cloud: { + properties: { + image: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + value: dataset, + }, + namespace: { + type: 'constant_keyword', + value: 'default', + }, + type: { + type: 'constant_keyword', + value: 'logs', + }, + }, + }, + ecs: { + properties: { + version: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + error: { + properties: { + message: { + type: 'match_only_text', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + module: { + type: 'constant_keyword', + value: 'nginx', + }, + }, + }, + host: { + properties: { + containerized: { + type: 'boolean', + }, + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + os: { + properties: { + build: { + type: 'keyword', + ignore_above: 1024, + }, + codename: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + input: { + properties: { + type: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + log: { + properties: { + level: { + type: 'keyword', + ignore_above: 1024, + }, + offset: { + type: 'long', + }, + }, + }, + network: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + nginx: { + properties: { + access: { + properties: { + remote_ip_list: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + }, + }, + test_field: { + type: 'keyword', + ignore_above: 1024, + }, + tls: { + properties: { + established: { + type: 'boolean', + }, + }, + }, + trace: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, +}); diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts index 712ed11a28f93..0d8d8e8865d52 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts @@ -385,6 +385,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataStream: degradedDataStreamName, }); + await PageObjects.datasetQuality.waitUntilTableLoaded(); + const rows = await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 4072dcec8a25c..59d58a3e83151 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -17,6 +17,7 @@ import { } from './data'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; +import { logsNginxMappings } from './custom_mappings/custom_integration_mappings'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects([ @@ -31,35 +32,43 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esClient = getService('es'); const retry = getService('retry'); const to = new Date().toISOString(); + const type = 'logs'; const degradedDatasetName = 'synth.degraded'; - const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const degradedDataStreamName = `${type}-${degradedDatasetName}-${defaultNamespace}`; const degradedDatasetWithLimitsName = 'synth.degraded.rca'; - const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`; + const degradedDatasetWithLimitDataStreamName = `${type}-${degradedDatasetWithLimitsName}-${defaultNamespace}`; const serviceName = 'test_service'; const count = 5; const customComponentTemplateName = 'logs-synth@mappings'; - describe('Degraded fields flyout', function () { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - await PageObjects.svlCommonPage.loginWithPrivilegedRole(); - }); - - after(async () => { - await synthtrace.clean(); - }); + const nginxAccessDatasetName = 'nginx.access'; + const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; + const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; + const nginxPkg = { + name: 'nginx', + version: '1.23.0', + }; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + await PageObjects.svlCommonPage.loginAsAdmin(); + }); + + after(async () => { + await synthtrace.clean(); + }); it('should open and close the flyout when user clicks on the expand button', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, @@ -88,30 +97,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - describe('testing root cause for ignored fields', () => { before(async () => { // Create custom component template @@ -140,8 +125,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { hidden: false, }, }); - // Ingest Degraded Logs with 25 fields + + // Install Nginx Integration and ingest logs for it + await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); + + // Create custom component template to avoid issues with LogsDB + await synthtrace.createComponentTemplate( + customComponentTemplateNameNginx, + logsNginxMappings(nginxAccessDatasetName) + ); + await synthtrace.index([ + // Ingest Degraded Logs with 25 fields timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -159,7 +154,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .defaults({ 'service.name': serviceName, 'trace.id': generateShortId(), - test_field: [MORE_THAN_1024_CHARS, 'hello world'], + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], }) .timestamp(timestamp) ); @@ -174,8 +192,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); - // Ingest Degraded Logs with 26 field + // Set Limit of 42 + await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { + 'mapping.total_fields.limit': 43, + }); + await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -194,7 +217,31 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 44 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -203,9 +250,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: degradedDatasetWithLimitsName, + }) + '-000002', + { + 'mapping.total_fields.limit': 26, + } + ); + + // Set Limit of 44 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: nginxAccessDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 44, + } + ); - // Ingest docs with 26 fields again await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -221,20 +289,164 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .service(serviceName) .namespace(defaultNamespace) .defaults({ - 'log.file.path': '/my-service.log', 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); }), ]); + await PageObjects.svlCommonPage.loginAsAdmin(); }); - describe('field character limit exceeded', () => { - it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(4); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(3); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + describe('character limit exceeded', () => { + it('should display cause as "field character limit exceeded" when a field is ignored due to character limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', @@ -251,25 +463,169 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.closeFlyout(); }); - it('should display values when cause is "field ignored"', async () => { + it('should display values when cause is "field character limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', }); await retry.tryForTime(5000, async () => { - const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + const testFieldValue1Exists = await PageObjects.datasetQuality.doesTextExist( 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', MORE_THAN_1024_CHARS ); - expect(testFieldValueExists).to.be(true); + const testFieldValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + expect(testFieldValue1Exists).to.be(true); + expect(testFieldValue2Exists).to.be(true); }); await PageObjects.datasetQuality.closeFlyout(); }); + + it('should display the maximum character limit when cause is "field character limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const limitValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-characterLimit', + '1024' + ); + expect(limitValueExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should show possible mitigation section with manual options for non integrations', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + // Should point to index template with the datastream name as value + expect(componentTemplateUrl).to.be( + `/data/index_management/templates/${degradedDatasetWithLimitDataStreamName}` + ); + + const nonIntegrationCustomName = `${type}@custom`; + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(nonIntegrationCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + nonIntegrationCustomName + )}` + ); + }); + + it('should show possible mitigation section with different manual options for integrations', async () => { + // Navigate to Integration Dataset + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'test_field', + }); + + await PageObjects.datasetQuality.waitUntilPossibleMitigationsLoaded(); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + const integrationSpecificCustomName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to component template with @custom as value + expect(componentTemplateUrl).to.be( + `/data/index_management/component_templates/${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(integrationSpecificCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${encodeURIComponent( + integrationSpecificCustomName + )}` + ); + }); }); - describe('field limit exceeded', () => { + describe('past field limit exceeded', () => { it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -287,7 +643,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.closeFlyout(); }); - it('should display the limit when the cause is "field limit exceeded"', async () => { + it('should display the current field limit when the cause is "field limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'cloud', @@ -317,122 +673,216 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('current quality issues', () => { - it('should display issues only from latest backing index when current issues toggle is on', async () => { + describe('current field limit issues', () => { + it('should display increase field limit as a possible mitigation for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - expect(currentIssuesToggleState).to.be(false); + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - expect(rows.length).to.eql(3); + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const linkURL = await linkButton.getAttribute('href'); + + expect(linkURL).to.be( + 'https://www.elastic.co/guide/en/elasticsearch/reference/master/mapping-settings-limit.html' ); + }); - const newCurrentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + it('should display increase field limit as a possible mitigation for non integration', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud.project', + }); - expect(newCurrentIssuesToggleState).to.be(true); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - const newRows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + // Should not display the panel to increase field limit + await testSubjects.missingOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); - expect(newRows.length).to.eql(2); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); }); - it('should keep the toggle on when url state says so', async () => { + it('should display additional input fields and button increasing the limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Should display current field limit + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingCurrentLimitFieldText'); - expect(currentIssuesToggleState).to.be(true); + const currentFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingCurrentLimitFieldText' + ); + + const currentFieldLimitValue = await currentFieldLimitInput.getAttribute('value'); + const currentFieldLimit = parseInt(currentFieldLimitValue as string, 10); + const currentFieldLimitDisabledStatus = await currentFieldLimitInput.getAttribute( + 'disabled' + ); + + expect(currentFieldLimit).to.be(44); + expect(currentFieldLimitDisabledStatus).to.be('true'); + + // Should display new field limit + await testSubjects.existOrFail( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitValue = await newFieldLimitInput.getAttribute('value'); + const newFieldLimit = parseInt(newFieldLimitValue as string, 10); + + // Should be 30% more the current limit + const newLimit = Math.round(currentFieldLimit * 1.3); + expect(newFieldLimit).to.be(newLimit); + + // Should display the apply button + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be(null); }); - it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + it('should validate input for new field limit', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Check value in Table - const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const countColumn = table['Docs count']; - expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + // Should not allow values less than current limit of 44 + await testSubjects.setValue( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText', + '42', + { + clearWithKeyboard: true, + typeCharByChar: true, + } + ); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const countValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '5' - ); - expect(countValue).to.be(true); + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be('true'); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + const invalidStatus = await newFieldLimitInput.getAttribute('aria-invalid'); + + expect(invalidStatus).to.be('true'); + }); + + it('should validate and show error callout when API call fails', async () => { + await PageObjects.svlCommonPage.loginWithPrivilegedRole(); + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Toggle the switch - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - // Check value in Table - const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const newCountColumn = newTable['Docs count']; - expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + await applyButton.click(); - // Check value in Flyout await retry.tryForTime(5000, async () => { - const newCountValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '15' - ); - expect(newCountValue).to.be(true); + // Should display the error callout + await testSubjects.existOrFail('datasetQualityDetailsNewFieldLimitErrorCallout'); }); + + await PageObjects.svlCommonPage.loginAsAdmin(); }); - it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + it('should let user increase the field limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - }); - it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', + await applyButton.click(); + + await retry.tryForTime(5000, async () => { + // Should display the success callout + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetSuccessCallout' + ); + + // Should display link to component template edited + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + + const ctLink = await testSubjects.find( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + const ctLinkURL = await ctLink.getAttribute('href'); + + const componentTemplateName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to the component template page + expect( + ctLinkURL?.endsWith( + `/data/index_management/component_templates/${encodeURIComponent( + componentTemplateName + )}` + ) + ).to.be(true); }); - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout - ); + // Refresh the time range to get the latest data + await PageObjects.datasetQuality.refreshDetailsPageData(); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + // The page should now handle this as ignore_malformed issue and show a warning + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist' ); + // Should not display the panel to increase field limit await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); }); }); @@ -443,6 +893,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { name: degradedDatasetWithLimitDataStreamName, }); await synthtrace.deleteComponentTemplate(customComponentTemplateName); + await PageObjects.observabilityLogsExplorer.uninstallPackage(nginxPkg); + await synthtrace.deleteComponentTemplate(customComponentTemplateNameNginx); }); }); });