[Dataset Quality] Add fix it flow for field limit (#195561)

## Summary

Closes - #190330

This PR implements the logic to support 

- One click increasing of Field Limit for Field Limit Issues (applicable
on for Integrations). For Non Integrations, only text is displayed as to
how they can do it.
- The One click increase updates the linked custom component template as
well as the last backing Index
- If Last Backing Index update fails due to any reason, it provides user
an option to trigger a Rollover manually.

## Demo

Not possible, to many things to display 😆 

## What's Pending ?

Tests

- [x] API tests
    - [x] Settings API
    - [x] Rollover API
    - [x] Apply New limit API
- [x] FTR tests
- [x] Displaying of various issues for integrations and non integrations
    - [x] Fix it Flow Good case, without Rollover
    - [x] Fix it Flow Good case, with Rollover
- [x] Manual Mitigation - Click on Component Template shold navigate to
proper logic based on Integration / Non
    - [x] Manual Mitigation - Ingest Pipeline
    - [x] Link for official Documentation
    
 ## How to setup a local environment
 
We will be setting up 2 different data streams, one with integration and
one without. Please follow the steps in the exact order
 
1. Start Local ES and Local Kibana
2. Install Nginx Integration 1st
3. Ingest data as per script here -
https://gist.github.com/achyutjhunjhunwala/03ea29190c6594544f584d2f0efa71e5
4. Set the Limit for the 2 datasets

```
PUT logs-synth.3-default/_settings
{
    "mapping.total_fields.limit": 36
}

// Set the limit for Nginx
PUT logs-nginx.access-default/_settings
{
    "mapping.total_fields.limit": 52
}
```

5. Now uncomment line number 59 from the synthtrace script to enable
cloud.project.id field and run the scenario again
6. Do a Rollover

```
POST logs-synth.3-default/_rollover
POST logs-nginx.access-default/_rollover
```

7. Get last backing index for both dataset

```
GET _data_stream/logs-synth.3-default/
GET _data_stream/logs-nginx.access-default
```

8. Increase the Limit by 1 but for last backing index

```
PUT .ds-logs-synth.3-default-2024.10.10-000002/_settings
{
    "mapping.total_fields.limit": 37
}

PUT .ds-logs-nginx.access-default-2024.10.10-000002/_settings
{
    "mapping.total_fields.limit": 53
}
```

9. Run the same Synthtrace scenario again.

This setup will give you 3 fields for testings

1. cloud.availability_zone - Which will show the character limit isue
2. cloud.project - Which will show an obsolete error which happened in
the past and now does not exists due to field limit
3. cloud.project.id - A current field limit issue

---------

Co-authored-by: Marco Antonio Ghiani <marcoantonio.ghiani01@gmail.com>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/packages/index-management/index_management_shared_types/src/types.ts

@@ -17,10 +17,21 @@ import type { LocatorPublic } from '@kbn/share-plugin/public';
 import { ExtensionsSetup } from './services/extensions_service';
 import { PublicApiServiceSetup } from './services/public_api_service';
 
-export interface IndexManagementLocatorParams extends SerializableRecord {
-  page: 'data_streams_details';
-  dataStreamName?: string;
-}
+export type IndexManagementLocatorParams = SerializableRecord &
+  (
+    | {
+        page: 'data_streams_details';
+        dataStreamName?: string;
+      }
+    | {
+        page: 'index_template';
+        indexTemplate: string;
+      }
+    | {
+        page: 'component_template';
+        componentTemplate: string;
+      }
+  );
 
 export type IndexManagementLocator = LocatorPublic<IndexManagementLocatorParams>;
 

x-pack/plugins/index_management/public/application/services/routing.ts

@@ -11,6 +11,7 @@ import { Section } from '../../../common/constants';
 import type { IndexDetailsTabId } from '../../../common/constants';
 import { ExtensionsService } from '../../services/extensions_service';
 import { IndexDetailsSection } from '../../../common/constants';
+
 export const getTemplateListLink = () => `/templates`;
 
 export const getTemplateDetailsLink = (name: string, isLegacy?: boolean) => {
@@ -81,6 +82,11 @@ export const getComponentTemplatesLink = (usedByTemplateName?: string) => {
   }
   return url;
 };
+
+export const getComponentTemplateDetailLink = (name: string) => {
+  return `/component_templates/${encodeURIComponent(name)}`;
+};
+
 export const navigateToIndexDetailsPage = (
   indexName: string,
   indicesListURLParams: string,

x-pack/plugins/index_management/public/locator.test.ts

@@ -34,4 +34,26 @@ describe('Index Management URL locator', () => {
     });
     expect(path).toBe('/data/index_management/data_streams/test');
   });
+
+  test('locator returns the correct url for index_template', async () => {
+    const indexTemplateName = 'test@custom';
+    const { path } = await locator.getLocation({
+      page: 'index_template',
+      indexTemplate: indexTemplateName,
+    });
+    expect(path).toBe(
+      encodeURI(`/data/index_management/templates/${encodeURIComponent(indexTemplateName)}`)
+    );
+  });
+
+  test('locator returns the correct url for component_template', async () => {
+    const componentTemplateName = 'log@custom';
+    const { path } = await locator.getLocation({
+      page: 'component_template',
+      componentTemplate: componentTemplateName,
+    });
+    expect(path).toBe(
+      `/data/index_management/component_templates/${encodeURIComponent(componentTemplateName)}`
+    );
+  });
 });

x-pack/plugins/index_management/public/locator.ts

@@ -8,7 +8,11 @@
 import { ManagementAppLocator } from '@kbn/management-plugin/common';
 import { LocatorDefinition } from '@kbn/share-plugin/public';
 import { IndexManagementLocatorParams } from '@kbn/index-management-shared-types';
-import { getDataStreamDetailsLink } from './application/services/routing';
+import {
+  getComponentTemplateDetailLink,
+  getDataStreamDetailsLink,
+  getTemplateDetailsLink,
+} from './application/services/routing';
 import { PLUGIN } from '../common/constants';
 
 export const INDEX_MANAGEMENT_LOCATOR_ID = 'INDEX_MANAGEMENT_LOCATOR_ID';
@@ -37,6 +41,18 @@ export class IndexManagementLocatorDefinition
           path: location.path + getDataStreamDetailsLink(params.dataStreamName!),
         };
       }
+      case 'index_template': {
+        return {
+          ...location,
+          path: location.path + getTemplateDetailsLink(params.indexTemplate),
+        };
+      }
+      case 'component_template': {
+        return {
+          ...location,
+          path: location.path + getComponentTemplateDetailLink(params.componentTemplate),
+        };
+      }
     }
   };
 }

x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts

@@ -134,22 +134,39 @@ export const degradedFieldAnalysisRt = rt.intersection([
       type: rt.string,
       ignore_above: rt.number,
     }),
+    defaultPipeline: rt.string,
   }),
 ]);
 
 export type DegradedFieldAnalysis = rt.TypeOf<typeof degradedFieldAnalysisRt>;
 
-export const dataStreamSettingsRt = rt.intersection([
+export const updateFieldLimitResponseRt = rt.intersection([
   rt.type({
-    lastBackingIndexName: rt.string,
+    isComponentTemplateUpdated: rt.union([rt.boolean, rt.undefined]),
+    isLatestBackingIndexUpdated: rt.union([rt.boolean, rt.undefined]),
+    customComponentTemplateName: rt.string,
   }),
   rt.partial({
-    createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless
-    integration: rt.string,
-    datasetUserPrivileges: datasetUserPrivilegesRt,
+    error: rt.string,
   }),
 ]);
 
+export type UpdateFieldLimitResponse = rt.TypeOf<typeof updateFieldLimitResponseRt>;
+
+export const dataStreamRolloverResponseRt = rt.type({
+  acknowledged: rt.boolean,
+});
+
+export type DataStreamRolloverResponse = rt.TypeOf<typeof dataStreamRolloverResponseRt>;
+
+export const dataStreamSettingsRt = rt.partial({
+  lastBackingIndexName: rt.string,
+  indexTemplate: rt.string,
+  createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless
+  integration: rt.string,
+  datasetUserPrivileges: datasetUserPrivilegesRt,
+});
+
 export type DataStreamSettings = rt.TypeOf<typeof dataStreamSettingsRt>;
 
 export const dataStreamDetailsRt = rt.partial({

x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts

@@ -14,3 +14,8 @@ export interface AnalyzeDegradedFieldsParams {
   lastBackingIndex: string;
   degradedField: string;
 }
+
+export interface UpdateFieldLimitParams {
+  dataStream: string;
+  newFieldLimit: number;
+}

x-pack/plugins/observability_solution/dataset_quality/common/translations.ts

@@ -500,3 +500,182 @@ export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate
       'This issue was detected in an older version of the dataset, but not in the most recent version.',
   }
 );
+
+export const possibleMitigationTitle = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigationTitle',
+  {
+    defaultMessage: 'Possible mitigation',
+  }
+);
+
+export const increaseFieldMappingLimitTitle = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.increaseFieldMappingLimitTitle',
+  {
+    defaultMessage: 'Increase field mapping limit',
+  }
+);
+
+export const fieldLimitMitigationDescriptionText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationDescription',
+  {
+    defaultMessage:
+      'The field mapping limit sets the maximum number of fields in an index. When exceeded, additional fields are ignored. To prevent this, increase your field mapping limit.',
+  }
+);
+
+export const fieldLimitMitigationConsiderationText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations',
+  {
+    defaultMessage: 'Before changing the field limit, consider the following:',
+  }
+);
+
+export const fieldLimitMitigationConsiderationText1 = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations1',
+  {
+    defaultMessage: 'Increasing the field limit could slow cluster performance.',
+  }
+);
+
+export const fieldLimitMitigationConsiderationText2 = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations2',
+  {
+    defaultMessage: 'Increasing the field limit also resolves field limit issues for other fields.',
+  }
+);
+
+export const fieldLimitMitigationConsiderationText3 = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations3',
+  {
+    defaultMessage:
+      'This change applies to the [name] component template and affects all namespaces in the template.',
+  }
+);
+
+export const fieldLimitMitigationConsiderationText4 = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations4',
+  {
+    defaultMessage:
+      'You need to roll over affected data streams to apply mapping changes to component templates.',
+  }
+);
+
+export const fieldLimitMitigationCurrentLimitLabelText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationCurrentLimitLabelText',
+  {
+    defaultMessage: 'Current limit',
+  }
+);
+
+export const fieldLimitMitigationNewLimitButtonText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitButtonText',
+  {
+    defaultMessage: 'New limit',
+  }
+);
+
+export const fieldLimitMitigationNewLimitPlaceholderText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitPlaceholderText',
+  {
+    defaultMessage: 'New field limit',
+  }
+);
+
+export const fieldLimitMitigationApplyButtonText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationApplyButtonText',
+  {
+    defaultMessage: 'Apply',
+  }
+);
+
+export const otherMitigationsLoadingAriaText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsLoadingText',
+  {
+    defaultMessage: 'Loading possible mitigations',
+  }
+);
+
+export const otherMitigationsCustomComponentTemplate = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomComponentTemplate',
+  {
+    defaultMessage: 'Add or edit custom component template',
+  }
+);
+
+export const otherMitigationsCustomIngestPipeline = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomIngestPipeline',
+  {
+    defaultMessage: 'Add or edit custom ingest pipeline',
+  }
+);
+
+export const fieldLimitMitigationOfficialDocumentation = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationOfficialDocumentation',
+  {
+    defaultMessage: 'Documentation',
+  }
+);
+
+export const fieldLimitMitigationSuccessMessage = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessMessage',
+  {
+    defaultMessage: 'New limit set!',
+  }
+);
+
+export const fieldLimitMitigationSuccessComponentTemplateLinkText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessComponentTemplateLinkText',
+  {
+    defaultMessage: 'See component template',
+  }
+);
+
+export const fieldLimitMitigationPartiallyFailedMessage = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessage',
+  {
+    defaultMessage: 'Changes not applied to new data',
+  }
+);
+
+export const fieldLimitMitigationFailedMessage = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessage',
+  {
+    defaultMessage: 'Changes not applied',
+  }
+);
+
+export const fieldLimitMitigationFailedMessageDescription = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessageDescription',
+  {
+    defaultMessage: 'Failed to set new limit',
+  }
+);
+
+export const fieldLimitMitigationPartiallyFailedMessageDescription = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessageDescription',
+  {
+    defaultMessage:
+      'The component template was successfully updated with the new field limit, but the changes were not applied to the most recent backing index. Perform a rollover to apply your changes to new data.',
+  }
+);
+
+export const fieldLimitMitigationRolloverButton = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationRolloverButton',
+  {
+    defaultMessage: 'Rollover',
+  }
+);
+
+export const manualMitigationCustomPipelineCopyPipelineNameAriaText = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.copyPipelineNameAriaText',
+  {
+    defaultMessage: 'Copy pipeline name',
+  }
+);
+
+export const manualMitigationCustomPipelineCreateEditPipelineLink = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.possibleMitigation.createEditPipelineLink',
+  {
+    defaultMessage: 'create or edit the pipeline',
+  }
+);

x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * There are index templates like this metrics-apm.service_transaction.10m@template which exists.
+ * Hence this @ needs to be removed to derive the custom component template name.
+ */
+export function getComponentTemplatePrefixFromIndexTemplate(indexTemplate: string) {
+  if (indexTemplate.includes('@')) {
+    return indexTemplate.split('@')[0];
+  }
+
+  return indexTemplate;
+}