Merge branch 'main' into adhoc_views_in_sourcerer

packages/kbn-apm-synthtrace/README.md

@@ -119,7 +119,7 @@ Scenario files accept 3 arguments, 2 of them optional and 1 mandatory
 |-------------|:----------|------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `generate`  | mandatory | This is the main function responsible for returning the events which will be indexed                                                                 |
 | `bootstrap`  | optional  | In case some setup needs to be done, before the data is generated, this function provides access to all available ES Clients to play with            |
-| `setClient` | optional  | By default the apmEsClient used to generate data. If anyother client like logsEsClient needs to be used instead, this is where it should be returned |
+| `teardown` | optional  | In case some setup needs to be done, after all data is generated, this function provides access to all available ES Clients to play with |
 
 The following options are supported:
 

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -42,4 +42,5 @@ type Generate<TFields> = (options: {
 export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
   bootstrap?: (options: EsClients & KibanaClients) => Promise<void>;
   generate: Generate<TFields>;
+  teardown?: (options: EsClients & KibanaClients) => Promise<void>;
 }>;

packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts

@@ -38,7 +38,11 @@ export async function startLiveDataUpload({
   } = await bootstrap(runOptions);
 
   const scenario = await getScenario({ file, logger });
-  const { generate, bootstrap: scenarioBootsrap } = await scenario({ ...runOptions, logger });
+  const {
+    generate,
+    bootstrap: scenarioBootsrap,
+    teardown: scenarioTearDown,
+  } = await scenario({ ...runOptions, logger });
 
   if (scenarioBootsrap) {
     await scenarioBootsrap({
@@ -59,11 +63,27 @@ export async function startLiveDataUpload({
   // @ts-expect-error upgrade typescript v4.9.5
   const cachedStreams: WeakMap<SynthtraceEsClient, PassThrough> = new WeakMap();
 
-  process.on('SIGINT', () => closeStreams());
-  process.on('SIGTERM', () => closeStreams());
-  process.on('SIGQUIT', () => closeStreams());
+  process.on('SIGINT', () => closeStreamsAndTeardown());
+  process.on('SIGTERM', () => closeStreamsAndTeardown());
+  process.on('SIGQUIT', () => closeStreamsAndTeardown());
+
+  async function closeStreamsAndTeardown() {
+    if (scenarioTearDown) {
+      try {
+        await scenarioTearDown({
+          apmEsClient,
+          logsEsClient,
+          infraEsClient,
+          otelEsClient,
+          syntheticsEsClient,
+          entitiesEsClient,
+          entitiesKibanaClient,
+        });
+      } catch (error) {
+        logger.error('Error during scenario teardown', error);
+      }
+    }
 
-  function closeStreams() {
     currentStreams.forEach((stream) => {
       stream.end(() => {
         process.exit(0);

packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts

@@ -84,7 +84,7 @@ async function start() {
 
   logger.info(`Running scenario from ${bucketFrom.toISOString()} to ${bucketTo.toISOString()}`);
 
-  const { generate, bootstrap } = await scenario({ ...runOptions, logger });
+  const { generate, bootstrap, teardown } = await scenario({ ...runOptions, logger });
 
   if (bootstrap) {
     await bootstrap({
@@ -142,6 +142,18 @@ async function start() {
 
     await Promise.all(promises);
   });
+
+  if (teardown) {
+    await teardown({
+      apmEsClient,
+      logsEsClient,
+      infraEsClient,
+      syntheticsEsClient,
+      otelEsClient,
+      entitiesEsClient,
+      entitiesKibanaClient,
+    });
+  }
 }
 
 parentPort!.on('message', (message) => {

packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts

@@ -52,6 +52,15 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
     }
   }
 
+  async deleteIndexTemplate(name: IndexTemplateName) {
+    try {
+      await this.client.indices.deleteIndexTemplate({ name });
+      this.logger.info(`Index template successfully deleted: ${name}`);
+    } catch (err) {
+      this.logger.error(`Index template deletion failed: ${name} - ${err.message}`);
+    }
+  }
+
   async createComponentTemplate({
     name,
     mappings,
@@ -150,6 +159,17 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
     }
   }
 
+  async deleteCustomPipeline(id = LogsCustom) {
+    try {
+      this.client.ingest.deletePipeline({
+        id,
+      });
+      this.logger.info(`Custom pipeline deleted: ${id}`);
+    } catch (err) {
+      this.logger.error(`Custom pipeline deletion failed: ${id} - ${err.message}`);
+    }
+  }
+
   getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
     return logsPipeline({ includeSerialization });
   }

packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts

@@ -36,6 +36,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
     bootstrap: async ({ logsEsClient }) => {
       if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
     },
+    teardown: async ({ logsEsClient }) => {
+      await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
+    },
     generate: ({ range, clients: { logsEsClient } }) => {
       const { logger } = runOptions;
 

packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts

@@ -75,6 +75,11 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
         },
       });
     },
+    teardown: async ({ logsEsClient }) => {
+      await logsEsClient.deleteComponentTemplate(LogsCustom);
+      await logsEsClient.deleteCustomPipeline();
+      if (isLogsDb) await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
+    },
     generate: ({ range, clients: { logsEsClient } }) => {
       const { logger } = runOptions;
 

packages/kbn-apm-synthtrace/src/scenarios/logs_and_metrics.ts

@@ -38,6 +38,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
     bootstrap: async ({ logsEsClient }) => {
       if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
     },
+    teardown: async ({ logsEsClient }) => {
+      await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
+    },
     generate: ({ range, clients: { logsEsClient, apmEsClient } }) => {
       const { numServices = 3 } = runOptions.scenarioOpts || {};
       const { logger } = runOptions;

packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts

@@ -52,6 +52,9 @@ const scenario: Scenario<LogDocument | InfraDocument | ApmFields> = async (runOp
     bootstrap: async ({ logsEsClient }) => {
       if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
     },
+    teardown: async ({ logsEsClient }) => {
+      await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
+    },
     generate: ({ range, clients: { logsEsClient, infraEsClient, apmEsClient } }) => {
       const {
         numSpaces,

packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts

@@ -70,6 +70,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
     bootstrap: async ({ logsEsClient }) => {
       if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
     },
+    teardown: async ({ logsEsClient }) => {
+      await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
+    },
     generate: ({ range, clients: { logsEsClient } }) => {
       const { logger } = runOptions;
 

packages/kbn-apm-synthtrace/src/scenarios/simple_non_ecs_logs.ts

@@ -78,6 +78,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
       await logsEsClient.createIndex('cloud-logs-synth.2-default');
       if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
     },
+    teardown: async ({ logsEsClient }) => {
+      await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
+    },
     generate: ({ range, clients: { logsEsClient } }) => {
       const { logger } = runOptions;
 

packages/kbn-apm-synthtrace/src/scenarios/unstructured_logs.ts

@@ -20,6 +20,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
     bootstrap: async ({ logsEsClient }) => {
       if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
     },
+    teardown: async ({ logsEsClient }) => {
+      await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
+    },
     generate: ({ range, clients: { logsEsClient } }) => {
       const { logger } = runOptions;
 

src/core/packages/http/server-internal/src/csp/config.ts

@@ -90,6 +90,10 @@ const configSchema = schema.object(
           defaultValue: [],
           validate: getDirectiveValidator({ allowNone: false, allowNonce: false }),
         }),
+        object_src: schema.arrayOf(schema.string(), {
+          defaultValue: [],
+          validate: getDirectiveValidator({ allowNone: true, allowNonce: false }),
+        }),
       })
     ),
     strict: schema.boolean({ defaultValue: true }),

src/core/packages/http/server-internal/src/csp/csp_config.test.ts

@@ -36,7 +36,7 @@ describe('CspConfig', () => {
         "disableEmbedding": false,
         "disableUnsafeEval": true,
         "header": "script-src 'report-sample' 'self'; worker-src 'report-sample' 'self' blob:; style-src 'report-sample' 'self' 'unsafe-inline'",
-        "reportOnlyHeader": "form-action 'report-sample' 'self'",
+        "reportOnlyHeader": "form-action 'report-sample' 'self'; object-src 'report-sample' 'none'",
         "strict": true,
         "warnLegacyBrowsers": true,
       }

src/core/packages/http/server-internal/src/csp/csp_directives.ts

@@ -22,13 +22,15 @@ export type CspDirectiveName =
   | 'img-src'
   | 'report-uri'
   | 'report-to'
-  | 'form-action';
+  | 'form-action'
+  | 'object-src';
 
 /**
  * The default report only directives rules
  */
 export const defaultReportOnlyRules: Partial<Record<CspDirectiveName, string[]>> = {
   'form-action': [`'report-sample'`, `'self'`],
+  'object-src': [`'report-sample'`, `'none'`],
 };
 
 /**
@@ -189,6 +191,10 @@ const parseConfigDirectives = (cspConfig: CspConfigType): CspConfigDirectives =>
     reportOnlyDirectives.set('form-action', cspConfig.report_only?.form_action);
   }
 
+  if (cspConfig.report_only?.object_src?.length) {
+    reportOnlyDirectives.set('object-src', cspConfig.report_only?.object_src);
+  }
+
   return {
     enforceDirectives,
     reportOnlyDirectives,

src/core/server/integration_tests/http/lifecycle.test.ts

@@ -1503,7 +1503,7 @@ describe('runs with default preResponse handlers', () => {
       `script-src 'report-sample' 'self' 'unsafe-eval'; worker-src 'report-sample' 'self' blob:; style-src 'report-sample' 'self' 'unsafe-inline'`
     );
     expect(response.header['content-security-policy-report-only']).toBe(
-      `form-action 'report-sample' 'self'`
+      `form-action 'report-sample' 'self'; object-src 'report-sample' 'none'`
     );
   });
 });

src/core/server/integration_tests/saved_objects/migrations/group3/multiple_es_nodes.test.ts

@@ -96,8 +96,7 @@ function createRoot({ logFileName, hosts }: RootConfig) {
   });
 }
 
-// FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/167676
-describe.skip('migration v2', () => {
+describe('migration v2', () => {
   let esServer: TestElasticsearchUtils;
   let root: Root;
   const migratedIndexAlias = `.kibana_${pkg.version}`;

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -40,6 +40,7 @@ kibana_vars=(
     csp.report_uri
     csp.report_to
     csp.report_only.form_action
+    csp.report_only.object_src
     permissionsPolicy.report_to
     data.autocomplete.valueSuggestions.terminateAfter
     data.autocomplete.valueSuggestions.timeout

src/platform/packages/shared/kbn-search-connectors/types/native_connectors.ts

@@ -4283,7 +4283,7 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
       },
       secret_value: {
         default_value: null,
-        depends_on: [],
+        depends_on: [{ field: 'auth_method', value: 'secret' }],
         display: TEXTBOX,
         label: translate(
           'searchConnectors.nativeConnectors.sharepoint_online.configuration.secretValueLabel',

x-pack/platform/plugins/shared/automatic_import/server/templates/processors/kv.yml.njk

@@ -1,7 +1,7 @@
   - kv:
       field: message
-      field_split: "{{ kvInput.field_split }}"
-      value_split: "{{ kvInput.value_split }}"
-      trim_key: "{{ kvInput.trim_key }}"
-      trim_value: "{{ kvInput.trim_value }}"
-      target_field: "{{ packageName }}.{{ dataStreamName }}"
+      field_split: '{{ kvInput.field_split }}'
+      value_split: '{{ kvInput.value_split }}'
+      trim_key: '{{ kvInput.trim_key }}'
+      trim_value: '{{ kvInput.trim_value }}'
+      target_field: '{{ packageName }}.{{ dataStreamName }}'

x-pack/solutions/observability/packages/utils_server/es/storage/index_adapter/index.ts

@@ -554,6 +554,7 @@ export class StorageIndexAdapter<TStorageSettings extends IndexStorageSettings,
           request: {} as unknown as DiagnosticResult['meta']['request'],
         },
         warnings: [],
+        body: 'resource_not_found_exception',
         statusCode: 404,
       });
     }

x-pack/solutions/observability/plugins/infra/public/alerting/log_threshold/components/expression_editor/editor.tsx

@@ -5,7 +5,14 @@
  * 2.0.
  */
 
-import { EuiButton, EuiCallOut, EuiLoadingSpinner, EuiSpacer } from '@elastic/eui';
+import {
+  EuiButton,
+  EuiCallOut,
+  EuiLink,
+  EuiLoadingSpinner,
+  EuiSpacer,
+  EuiText,
+} from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import type { FC, PropsWithChildren } from 'react';
 import React, { useCallback, useMemo, useState } from 'react';
@@ -101,7 +108,6 @@ export const ExpressionEditor: React.FC<
   const {
     services: { logsShared },
   } = useKibanaContextForPlugin(); // injected during alert registration
-
   return (
     <>
       {isInternal ? (
@@ -164,6 +170,9 @@ export const Editor: React.FC<RuleTypeParamsExpressionProps<PartialRuleParams, L
   const { setRuleParams, ruleParams, errors } = props;
   const [hasSetDefaults, setHasSetDefaults] = useState<boolean>(false);
   const { logViewReference, resolvedLogView } = useLogViewContext();
+  const {
+    services: { http },
+  } = useKibanaContextForPlugin();
 
   if (logViewReference.type !== 'log-view-reference') {
     throw new Error('The Log Threshold rule type only supports persisted Log Views');
@@ -289,34 +298,41 @@ export const Editor: React.FC<RuleTypeParamsExpressionProps<PartialRuleParams, L
   return (
     <>
       {resolvedLogView && <LogViewSwitcher logView={resolvedLogView} />}
-
+      <EuiText size="xs">
+        {i18n.translate('xpack.infra.editor.modifyLogViewSetting', {
+          defaultMessage: 'To modify go to ',
+        })}
+        <EuiLink
+          data-test-subj="infraEditorLinkToAdvancedSettings"
+          href={http.basePath.prepend('/app/management/kibana/settings?query=Log+sources')}
+        >
+          {i18n.translate('xpack.infra.editor.euiLink.advancedSettingsLabel', {
+            defaultMessage: 'Advanced Settings.',
+          })}
+        </EuiLink>
+      </EuiText>
+      <EuiSpacer size="m" />
       <TypeSwitcher criteria={ruleParams.criteria || []} updateType={updateType} />
-
       {ruleParams.criteria && !isRatioRule(ruleParams.criteria) && criteriaComponent}
-
       <Threshold
         comparator={ruleParams.count?.comparator}
         value={ruleParams.count?.value}
         updateThreshold={updateThreshold}
         errors={thresholdErrors}
       />
-
       <ForLastExpression
         timeWindowSize={ruleParams.timeSize}
         timeWindowUnit={ruleParams.timeUnit}
         onChangeWindowSize={updateTimeSize}
         onChangeWindowUnit={updateTimeUnit}
         errors={{ timeWindowSize: timeWindowSizeErrors, timeSizeUnit: timeSizeUnitErrors }}
       />
-
       <GroupByExpression
         selectedGroups={ruleParams.groupBy}
         onChange={updateGroupBy}
         fields={groupByFields}
       />
-
       {ruleParams.criteria && isRatioRule(ruleParams.criteria) && criteriaComponent}
-
       {shouldShowGroupByOptimizationWarning && (
         <>
           <EuiSpacer size="l" />
@@ -331,7 +347,6 @@ export const Editor: React.FC<RuleTypeParamsExpressionProps<PartialRuleParams, L
           </EuiCallOut>
         </>
       )}
-
       <EuiSpacer size="l" />
     </>
   );