Make expectSnapshot available in all functional test runs

Closes #80292.

packages/kbn-dev-utils/src/run/flags.ts

@@ -29,7 +29,6 @@ export interface Flags {
   help: boolean;
   _: string[];
   unexpected: string[];
-
   [key: string]: undefined | boolean | string | string[];
 }
 

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -61,6 +61,7 @@ export function runFtrCli() {
             exclude: toArray(flags['exclude-tag'] as string | string[]),
           },
           updateBaselines: flags.updateBaselines,
+          updateSnapshots: flags.updateSnapshots,
         }
       );
 
@@ -126,7 +127,15 @@ export function runFtrCli() {
           'exclude-tag',
           'kibana-install-dir',
         ],
-        boolean: ['bail', 'invert', 'test-stats', 'updateBaselines', 'throttle', 'headless'],
+        boolean: [
+          'bail',
+          'invert',
+          'test-stats',
+          'updateBaselines',
+          'throttle',
+          'headless',
+          'updateSnapshots',
+        ],
         default: {
           config: 'test/functional/config.js',
         },
@@ -141,6 +150,7 @@ export function runFtrCli() {
         --exclude-tag=tag  a tag to be excluded, pass multiple times for multiple tags
         --test-stats       print the number of tests (included and excluded) to STDERR
         --updateBaselines  replace baseline screenshots with whatever is generated from the test
+        --updateSnapshots  replace inline and file snapshots with whatever is generated from the test
         --kibana-install-dir  directory where the Kibana install being tested resides
         --throttle         enable network throttling in Chrome browser
         --headless         run browser in headless mode

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -138,7 +138,7 @@ export const schema = Joi.object()
       .default(),
 
     updateBaselines: Joi.boolean().default(false),
-
+    updateSnapshots: Joi.boolean().default(false),
     browser: Joi.object()
       .keys({
         type: Joi.string().valid('chrome', 'firefox', 'msedge').default('chrome'),

packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js

@@ -21,6 +21,7 @@ import { isAbsolute } from 'path';
 
 import { loadTracer } from '../load_tracer';
 import { decorateMochaUi } from './decorate_mocha_ui';
+import { decorateSnapshotUi } from '../snapshots/decorate_snapshot_ui';
 
 /**
  *  Load an array of test files into a mocha instance
@@ -31,7 +32,17 @@ import { decorateMochaUi } from './decorate_mocha_ui';
  *  @param  {String} path
  *  @return {undefined} - mutates mocha, no return value
  */
-export const loadTestFiles = ({ mocha, log, lifecycle, providers, paths, updateBaselines }) => {
+export const loadTestFiles = ({
+  mocha,
+  log,
+  lifecycle,
+  providers,
+  paths,
+  updateBaselines,
+  updateSnapshots,
+}) => {
+  decorateSnapshotUi(lifecycle, updateSnapshots);
+
   const innerLoadTestFile = (path) => {
     if (typeof path !== 'string' || !isAbsolute(path)) {
       throw new TypeError('loadTestFile() only accepts absolute paths');

packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -53,6 +53,7 @@ export async function setupMocha(lifecycle, log, config, providers) {
     providers,
     paths: config.get('testFiles'),
     updateBaselines: config.get('updateBaselines'),
+    updateSnapshots: config.get('updateSnapshots'),
   });
 
   // Each suite has a tag that is the path relative to the root of the repo

packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.test.ts

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+describe('decorateSnapshotUi', () => {
+  describe('when running a test', () => {
+    beforeEach(() => {
+      jest.mock('fs');
+    });
+    it('passes when the snapshot matches the actual value', () => {});
+
+    it('throws when the snapshot does not match the actual value', () => {});
+  });
+});

x-pack/test/apm_api_integration/common/match_snapshot.ts → packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.ts

@@ -1,7 +1,20 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 import {
@@ -15,7 +28,8 @@ import expect from '@kbn/expect';
 import prettier from 'prettier';
 import babelTraverse from '@babel/traverse';
 import { Suite, Test } from 'mocha';
-import { flatten } from 'lodash';
+import { flatten, once } from 'lodash';
+import { Lifecycle } from 'packages/kbn-test/types/ftr';
 
 type ISnapshotState = InstanceType<typeof SnapshotState>;
 
@@ -59,12 +73,38 @@ function getSnapshotMeta(currentTest: Test) {
   };
 }
 
-export function registerMochaHooksForSnapshots() {
+const modifyStackTracePrepareOnce = once(() => {
+  const originalPrepareStackTrace = Error.prepareStackTrace;
+
+  // jest-snapshot uses a stack trace to determine which file/line/column
+  // an inline snapshot should be written to. We filter out match_snapshot
+  // from the stack trace to prevent it from wanting to write to this file.
+
+  Error.prepareStackTrace = (error, structuredStackTrace) => {
+    let filteredStrackTrace: NodeJS.CallSite[] = structuredStackTrace;
+    if (registered) {
+      filteredStrackTrace = filteredStrackTrace.filter((callSite) => {
+        // use .js as this file will be compiled
+        return !callSite.getFileName()?.endsWith('decorate_snapshot_ui.js');
+      });
+    }
+
+    if (originalPrepareStackTrace) {
+      return originalPrepareStackTrace(error, filteredStrackTrace);
+    }
+  };
+});
+
+export function decorateSnapshotUi(lifecycle: Lifecycle, updateSnapshots: boolean) {
   let snapshotStatesByFilePath: Record<
     string,
     { snapshotState: ISnapshotState; testsInFile: Test[] }
   > = {};
 
+  registered = true;
+
+  modifyStackTracePrepareOnce();
+
   addSerializer({
     serialize: (num: number) => {
       return String(parseFloat(num.toPrecision(15)));
@@ -74,15 +114,14 @@ export function registerMochaHooksForSnapshots() {
     },
   });
 
-  registered = true;
-
-  beforeEach(function () {
-    const currentTest = this.currentTest!;
+  // @ts-expect-error
+  global.expectSnapshot = expectSnapshot;
 
+  lifecycle.beforeEachTest.add((currentTest: Test) => {
     const { file, snapshotTitle } = getSnapshotMeta(currentTest);
 
     if (!snapshotStatesByFilePath[file]) {
-      snapshotStatesByFilePath[file] = getSnapshotState(file, currentTest);
+      snapshotStatesByFilePath[file] = getSnapshotState(file, currentTest, updateSnapshots);
     }
 
     testContext = {
@@ -95,17 +134,11 @@ export function registerMochaHooksForSnapshots() {
     };
   });
 
-  afterEach(function () {
-    testContext = null;
-  });
-
-  after(function () {
+  lifecycle.afterTestSuite.add(function () {
     // save snapshot after tests complete
 
     const unused: string[] = [];
 
-    const isUpdatingSnapshots = process.env.UPDATE_SNAPSHOTS;
-
     Object.keys(snapshotStatesByFilePath).forEach((file) => {
       const { snapshotState, testsInFile } = snapshotStatesByFilePath[file];
 
@@ -118,7 +151,7 @@ export function registerMochaHooksForSnapshots() {
         }
       });
 
-      if (!isUpdatingSnapshots) {
+      if (!updateSnapshots) {
         unused.push(...snapshotState.getUncheckedKeys());
       } else {
         snapshotState.removeUncheckedKeys();
@@ -136,31 +169,14 @@ export function registerMochaHooksForSnapshots() {
     }
 
     snapshotStatesByFilePath = {};
-
-    registered = false;
   });
 }
 
-const originalPrepareStackTrace = Error.prepareStackTrace;
-
-// jest-snapshot uses a stack trace to determine which file/line/column
-// an inline snapshot should be written to. We filter out match_snapshot
-// from the stack trace to prevent it from wanting to write to this file.
-
-Error.prepareStackTrace = (error, structuredStackTrace) => {
-  const filteredStrackTrace = structuredStackTrace.filter((callSite) => {
-    return !callSite.getFileName()?.endsWith('match_snapshot.ts');
-  });
-  if (originalPrepareStackTrace) {
-    return originalPrepareStackTrace(error, filteredStrackTrace);
-  }
-};
-
 function recursivelyGetTestsFromSuite(suite: Suite): Test[] {
   return suite.tests.concat(flatten(suite.suites.map((s) => recursivelyGetTestsFromSuite(s))));
 }
 
-function getSnapshotState(file: string, test: Test) {
+function getSnapshotState(file: string, test: Test, updateSnapshots: boolean) {
   const dirname = path.dirname(file);
   const filename = path.basename(file);
 
@@ -177,7 +193,7 @@ function getSnapshotState(file: string, test: Test) {
   const snapshotState = new SnapshotState(
     path.join(dirname + `/__snapshots__/` + filename.replace(path.extname(filename), '.snap')),
     {
-      updateSnapshot: process.env.UPDATE_SNAPSHOTS ? 'all' : 'new',
+      updateSnapshot: updateSnapshots ? 'all' : 'new',
       getPrettier: () => prettier,
       getBabelTraverse: () => babelTraverse,
     }

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -46,6 +46,9 @@ const options = {
   updateBaselines: {
     desc: 'Replace baseline screenshots with whatever is generated from the test.',
   },
+  updateSnapshots: {
+    desc: 'replace inline and file snapshots with whatever is generated from the test.',
+  },
   include: {
     arg: '<file>',
     desc: 'Files that must included to be run, can be included multiple times.',

packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js

@@ -76,6 +76,11 @@ describe('process options for run tests CLI', () => {
     expect(options).toMatchSnapshot();
   });
 
+  it('accepts boolean value for updateSnapshots', () => {
+    const options = processOptions({ updateSnapshots: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
   it('accepts source value for esFrom', () => {
     const options = processOptions({ esFrom: 'source' }, ['foo']);
     expect(options).toMatchSnapshot();

packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js

@@ -125,6 +125,14 @@ describe('run tests CLI', () => {
       expect(exitMock).not.toHaveBeenCalledWith();
     });
 
+    it('accepts boolean value for updateSnapshots', async () => {
+      global.process.argv.push('--updateSnapshots');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalledWith();
+    });
+
     it('accepts source value for esFrom', async () => {
       global.process.argv.push('--esFrom', 'source');
 

packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js

@@ -126,6 +126,15 @@ describe('start servers CLI', () => {
       checkMockConsoleLogSnapshot(logMock);
     });
 
+    it('accepts boolean value for updateSnapshots', async () => {
+      global.process.argv.push('--updateSnapshots');
+
+      await startServersCli('foo');
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      checkMockConsoleLogSnapshot(logMock);
+    });
+
     it('accepts source value for esFrom', async () => {
       global.process.argv.push('--esFrom', 'source');
 

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -22,7 +22,7 @@ import { CliError } from './run_cli';
 
 async function createFtr({
   configPath,
-  options: { installDir, log, bail, grep, updateBaselines, suiteFiles, suiteTags },
+  options: { installDir, log, bail, grep, updateBaselines, suiteFiles, suiteTags, updateSnapshots },
 }) {
   const config = await readConfigFile(log, configPath);
 
@@ -37,6 +37,7 @@ async function createFtr({
         installDir,
       },
       updateBaselines,
+      updateSnapshots,
       suiteFiles: {
         include: [...suiteFiles.include, ...config.get('suiteFiles.include')],
         exclude: [...suiteFiles.exclude, ...config.get('suiteFiles.exclude')],

x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts

@@ -6,7 +6,6 @@
 
 import expect from '@kbn/expect';
 import { format } from 'url';
-import { expectSnapshot } from '../../../common/match_snapshot';
 import { PromiseReturnType } from '../../../../../plugins/apm/typings/common';
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
 import archives_metadata from '../../../common/archives_metadata';

x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts

@@ -8,7 +8,6 @@ import expect from '@kbn/expect';
 import { format } from 'url';
 import { PromiseReturnType } from '../../../../../plugins/apm/typings/common';
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
-import { expectSnapshot } from '../../../common/match_snapshot';
 import archives_metadata from '../../../common/archives_metadata';
 
 export default function ApiTest({ getService }: FtrProviderContext) {

x-pack/test/apm_api_integration/basic/tests/index.ts

@@ -4,12 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { FtrProviderContext } from '../../common/ftr_provider_context';
-import { registerMochaHooksForSnapshots } from '../../common/match_snapshot';
 
 export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderContext) {
   describe('APM specs (basic)', function () {
-    registerMochaHooksForSnapshots();
-
     this.tags('ciGroup1');
 
     loadTestFile(require.resolve('./feature_controls'));

x-pack/test/apm_api_integration/basic/tests/metrics_charts/metrics_charts.ts

@@ -8,7 +8,6 @@ import { first } from 'lodash';
 import { MetricsChartsByAgentAPIResponse } from '../../../../../plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent';
 import { GenericMetricsChart } from '../../../../../plugins/apm/server/lib/metrics/transform_metrics_chart';
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
-import { expectSnapshot } from '../../../common/match_snapshot';
 
 interface ChartResponse {
   body: MetricsChartsByAgentAPIResponse;

x-pack/test/apm_api_integration/basic/tests/observability_overview/observability_overview.ts

@@ -5,7 +5,6 @@
  */
 import expect from '@kbn/expect';
 import archives_metadata from '../../../common/archives_metadata';
-import { expectSnapshot } from '../../../common/match_snapshot';
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
 
 export default function ApiTest({ getService }: FtrProviderContext) {