Make expectSnapshot available in all functional test runs

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -61,6 +61,7 @@ export function runFtrCli() {
             exclude: toArray(flags['exclude-tag'] as string | string[]),
           },
           updateBaselines: flags.updateBaselines,
+          updateSnapshots: flags.updateSnapshots,
         }
       );
 
@@ -126,7 +127,15 @@ export function runFtrCli() {
           'exclude-tag',
           'kibana-install-dir',
         ],
-        boolean: ['bail', 'invert', 'test-stats', 'updateBaselines', 'throttle', 'headless'],
+        boolean: [
+          'bail',
+          'invert',
+          'test-stats',
+          'updateBaselines',
+          'throttle',
+          'headless',
+          'updateSnapshots',
+        ],
         default: {
           config: 'test/functional/config.js',
         },
@@ -141,6 +150,7 @@ export function runFtrCli() {
         --exclude-tag=tag  a tag to be excluded, pass multiple times for multiple tags
         --test-stats       print the number of tests (included and excluded) to STDERR
         --updateBaselines  replace baseline screenshots with whatever is generated from the test
+        --updateSnapshots  replace inline and file snapshots with whatever is generated from the test
         --kibana-install-dir  directory where the Kibana install being tested resides
         --throttle         enable network throttling in Chrome browser
         --headless         run browser in headless mode

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -138,7 +138,7 @@ export const schema = Joi.object()
       .default(),
 
     updateBaselines: Joi.boolean().default(false),
-
+    updateSnapshots: Joi.boolean().default(false),
     browser: Joi.object()
       .keys({
         type: Joi.string().valid('chrome', 'firefox', 'msedge').default('chrome'),

packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js

@@ -21,6 +21,7 @@ import { isAbsolute } from 'path';
 
 import { loadTracer } from '../load_tracer';
 import { decorateMochaUi } from './decorate_mocha_ui';
+import { decorateSnapshotUi } from '../snapshots/decorate_snapshot_ui';
 
 /**
  *  Load an array of test files into a mocha instance
@@ -31,7 +32,19 @@ import { decorateMochaUi } from './decorate_mocha_ui';
  *  @param  {String} path
  *  @return {undefined} - mutates mocha, no return value
  */
-export const loadTestFiles = ({ mocha, log, lifecycle, providers, paths, updateBaselines }) => {
+export const loadTestFiles = ({
+  mocha,
+  log,
+  lifecycle,
+  providers,
+  paths,
+  updateBaselines,
+  updateSnapshots,
+}) => {
+  console.log('loadTestFiles');
+
+  decorateSnapshotUi(lifecycle, updateSnapshots);
+
   const innerLoadTestFile = (path) => {
     if (typeof path !== 'string' || !isAbsolute(path)) {
       throw new TypeError('loadTestFile() only accepts absolute paths');

packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -53,6 +53,7 @@ export async function setupMocha(lifecycle, log, config, providers) {
     providers,
     paths: config.get('testFiles'),
     updateBaselines: config.get('updateBaselines'),
+    updateSnapshots: config.get('updateSnapshots'),
   });
 
   // Each suite has a tag that is the path relative to the root of the repo

packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.test.ts

@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Test } from 'mocha';
+import { Lifecycle } from '../lifecycle';
+import { decorateSnapshotUi, expectSnapshot } from './decorate_snapshot_ui';
+import path from 'path';
+import fs from 'fs';
+
+describe('decorateSnapshotUi', () => {
+  describe('when running a test', () => {
+    let lifecycle: Lifecycle;
+    beforeEach(() => {
+      lifecycle = new Lifecycle();
+      decorateSnapshotUi(lifecycle, false);
+    });
+
+    it('passes when the snapshot matches the actual value', async () => {
+      const test: Test = {
+        title: 'Test',
+        file: 'foo.ts',
+        parent: {
+          file: 'foo.ts',
+          tests: [],
+          suites: [],
+        },
+      } as any;
+
+      await lifecycle.beforeEachTest.trigger(test);
+
+      expect(() => {
+        expectSnapshot('foo').toMatchInline(`"foo"`);
+      }).not.toThrow();
+    });
+
+    it('throws when the snapshot does not match the actual value', async () => {
+      const test: Test = {
+        title: 'Test',
+        file: 'foo.ts',
+        parent: {
+          file: 'foo.ts',
+          tests: [],
+          suites: [],
+        },
+      } as any;
+
+      await lifecycle.beforeEachTest.trigger(test);
+
+      expect(() => {
+        expectSnapshot('foo').toMatchInline(`"bar"`);
+      }).toThrow();
+    });
+
+    it('writes a snapshot to an external file if it does not exist', async () => {
+      const test: Test = {
+        title: 'Test',
+        file: __filename,
+        isPassed: () => true,
+      } as any;
+
+      // @ts-expect-error
+      test.parent = {
+        file: __filename,
+        tests: [test],
+        suites: [],
+      };
+
+      await lifecycle.beforeEachTest.trigger(test);
+
+      const snapshotFile = path.resolve(
+        __dirname,
+        '__snapshots__',
+        'decorate_snapshot_ui.test.snap'
+      );
+
+      expect(fs.existsSync(snapshotFile)).toBe(false);
+
+      expect(() => {
+        expectSnapshot('foo').toMatch();
+      }).not.toThrow();
+
+      await lifecycle.afterTestSuite.trigger(parent);
+
+      expect(fs.existsSync(snapshotFile)).toBe(true);
+
+      fs.unlinkSync(snapshotFile);
+
+      fs.rmdirSync(path.resolve(__dirname, '__snapshots__'));
+    });
+  });
+
+  describe('when updating snapshots', () => {
+    let lifecycle: Lifecycle;
+    beforeEach(() => {
+      lifecycle = new Lifecycle();
+      decorateSnapshotUi(lifecycle, true);
+    });
+
+    it("does'nt throw if the value does not match", async () => {
+      const test: Test = {
+        title: 'Test',
+        file: 'foo.ts',
+        parent: {
+          file: 'foo.ts',
+          tests: [],
+          suites: [],
+        },
+      } as any;
+
+      await lifecycle.beforeEachTest.trigger(test);
+
+      expect(() => {
+        expectSnapshot('bar').toMatchInline(`"foo"`);
+      }).not.toThrow();
+    });
+  });
+});

x-pack/test/apm_api_integration/common/match_snapshot.ts → packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.ts

@@ -1,7 +1,20 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
 
 import {
@@ -15,7 +28,8 @@ import expect from '@kbn/expect';
 import prettier from 'prettier';
 import babelTraverse from '@babel/traverse';
 import { Suite, Test } from 'mocha';
-import { flatten } from 'lodash';
+import { flatten, once } from 'lodash';
+import { Lifecycle } from 'packages/kbn-test/types/ftr';
 
 type ISnapshotState = InstanceType<typeof SnapshotState>;
 
@@ -59,12 +73,38 @@ function getSnapshotMeta(currentTest: Test) {
   };
 }
 
-export function registerMochaHooksForSnapshots() {
+const modifyStackTracePrepareOnce = once(() => {
+  const originalPrepareStackTrace = Error.prepareStackTrace;
+
+  // jest-snapshot uses a stack trace to determine which file/line/column
+  // an inline snapshot should be written to. We filter out match_snapshot
+  // from the stack trace to prevent it from wanting to write to this file.
+
+  Error.prepareStackTrace = (error, structuredStackTrace) => {
+    let filteredStrackTrace: NodeJS.CallSite[] = structuredStackTrace;
+    if (registered) {
+      filteredStrackTrace = filteredStrackTrace.filter((callSite) => {
+        // check for both compiled and uncompiled files
+        return !callSite.getFileName()?.match(/decorate_snapshot_ui\.(js|ts)/);
+      });
+    }
+
+    if (originalPrepareStackTrace) {
+      return originalPrepareStackTrace(error, filteredStrackTrace);
+    }
+  };
+});
+
+export function decorateSnapshotUi(lifecycle: Lifecycle, updateSnapshots: boolean) {
   let snapshotStatesByFilePath: Record<
     string,
     { snapshotState: ISnapshotState; testsInFile: Test[] }
   > = {};
 
+  registered = true;
+
+  modifyStackTracePrepareOnce();
+
   addSerializer({
     serialize: (num: number) => {
       return String(parseFloat(num.toPrecision(15)));
@@ -74,15 +114,14 @@ export function registerMochaHooksForSnapshots() {
     },
   });
 
-  registered = true;
-
-  beforeEach(function () {
-    const currentTest = this.currentTest!;
+  // @ts-expect-error
+  global.expectSnapshot = expectSnapshot;
 
+  lifecycle.beforeEachTest.add((currentTest: Test) => {
     const { file, snapshotTitle } = getSnapshotMeta(currentTest);
 
     if (!snapshotStatesByFilePath[file]) {
-      snapshotStatesByFilePath[file] = getSnapshotState(file, currentTest);
+      snapshotStatesByFilePath[file] = getSnapshotState(file, currentTest, updateSnapshots);
     }
 
     testContext = {
@@ -95,17 +134,14 @@ export function registerMochaHooksForSnapshots() {
     };
   });
 
-  afterEach(function () {
-    testContext = null;
-  });
-
-  after(function () {
-    // save snapshot after tests complete
+  lifecycle.afterTestSuite.add(function (testSuite) {
+    // save snapshot & check unused after top-level test suite completes
+    if (testSuite.parent?.parent) {
+      return;
+    }
 
     const unused: string[] = [];
 
-    const isUpdatingSnapshots = process.env.UPDATE_SNAPSHOTS;
-
     Object.keys(snapshotStatesByFilePath).forEach((file) => {
       const { snapshotState, testsInFile } = snapshotStatesByFilePath[file];
 
@@ -118,7 +154,7 @@ export function registerMochaHooksForSnapshots() {
         }
       });
 
-      if (!isUpdatingSnapshots) {
+      if (!updateSnapshots) {
         unused.push(...snapshotState.getUncheckedKeys());
       } else {
         snapshotState.removeUncheckedKeys();
@@ -131,36 +167,19 @@ export function registerMochaHooksForSnapshots() {
       throw new Error(
         `${unused.length} obsolete snapshot(s) found:\n${unused.join(
           '\n\t'
-        )}.\n\nRun tests again with \`UPDATE_SNAPSHOTS=1\` to remove them.`
+        )}.\n\nRun tests again with \`--updateSnapshots\` to remove them.`
       );
     }
 
     snapshotStatesByFilePath = {};
-
-    registered = false;
   });
 }
 
-const originalPrepareStackTrace = Error.prepareStackTrace;
-
-// jest-snapshot uses a stack trace to determine which file/line/column
-// an inline snapshot should be written to. We filter out match_snapshot
-// from the stack trace to prevent it from wanting to write to this file.
-
-Error.prepareStackTrace = (error, structuredStackTrace) => {
-  const filteredStrackTrace = structuredStackTrace.filter((callSite) => {
-    return !callSite.getFileName()?.endsWith('match_snapshot.ts');
-  });
-  if (originalPrepareStackTrace) {
-    return originalPrepareStackTrace(error, filteredStrackTrace);
-  }
-};
-
 function recursivelyGetTestsFromSuite(suite: Suite): Test[] {
   return suite.tests.concat(flatten(suite.suites.map((s) => recursivelyGetTestsFromSuite(s))));
 }
 
-function getSnapshotState(file: string, test: Test) {
+function getSnapshotState(file: string, test: Test, updateSnapshots: boolean) {
   const dirname = path.dirname(file);
   const filename = path.basename(file);
 
@@ -177,7 +196,7 @@ function getSnapshotState(file: string, test: Test) {
   const snapshotState = new SnapshotState(
     path.join(dirname + `/__snapshots__/` + filename.replace(path.extname(filename), '.snap')),
     {
-      updateSnapshot: process.env.UPDATE_SNAPSHOTS ? 'all' : 'new',
+      updateSnapshot: updateSnapshots ? 'all' : 'new',
       getPrettier: () => prettier,
       getBabelTraverse: () => babelTraverse,
     }