feat(integ-runner): add missing features from the integ manifest

INTEGRATION_TESTS.md

@@ -10,6 +10,7 @@ on what type of changes require integrations tests and how you should write inte
   - [New L2 Constructs](#new-l2-constructs)
   - [Existing L2 Constructs](#existing-l2-constructs)
   - [Assertions](#assertions)
+- [Running Integration Tests](#running-integration-tests)
 
 ## What are CDK Integration Tests
 
@@ -223,3 +224,40 @@ to deploy the Lambda Function _and_ then rerun the assertions to ensure that the
 
 ### Assertions
 ...Coming soon...
+
+## Running Integration Tests
+
+Most of the time you will only need to run integration tests for an individual module (i.e. `aws-lambda`). Other times you may need to run tests across multiple modules.
+In this case I would recommend running from the root directory like below.
+
+_Run snapshot tests only_
+```bash
+yarn integ-runner --directory packages/@aws-cdk
+```
+
+_Run snapshot tests and then re-run integration tests for failed snapshots_
+```bash
+yarn integ-runner --directory packages/@aws-cdk --update-on-failed
+```
+
+One benefit of running from the root directory like this is that it will only collect tests from "built" modules. If you have built the entire
+repo it will run all integration tests, but if you have only built a couple modules it will only run tests from those.
+
+### Running large numbers of Tests
+
+If you need to re-run a large number of tests you can run them in parallel like this.
+
+```bash
+yarn integ-runner --directory packages/@aws-cdk --update-on-failed \
+  --parallel-regions us-east-1 \
+  --parallel-regions us-east-2 \
+  --parallel-regions us-west-2 \
+  --parallel-regions eu-west-1 \
+  --profiles profile1 \
+  --profiles profile2 \
+  --profiles profile3 \
+  --verbose
+```
+
+When using both `--parallel-regions` and `--profiles` it will execute (regions*profiles) tests in parallel (in this example 12)
+If you want to execute more than 16 tests in parallel you can pass a higher value to `--max-workers`.

packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.version.json

@@ -1 +1 @@
-{"version":"17.0.0"}
+{"version":"17.0.0"}

packages/@aws-cdk/cloud-assembly-schema/schema/integ.schema.json

@@ -471,4 +471,4 @@
         }
     },
     "$schema": "http://json-schema.org/draft-07/schema#"
-}
+}

packages/@aws-cdk/integ-runner/README.md

@@ -44,9 +44,7 @@ to be a self contained CDK app. The runner will execute the following for each f
 - `--clean` (default=`true`)
   Destroy stacks after deploy (use `--no-clean` for debugging)
 - `--verbose` (default=`false`)
-  verbose logging
-- `--parallel` (default=`true`)
-  Run tests in parallel across default regions
+  verbose logging, including integration test metrics
 - `--parallel-regions` (default=`us-east-1`,`us-east-2`, `us-west-2`)
   List of regions to run tests in. If this is provided then all tests will
   be run in parallel across these regions
@@ -66,11 +64,18 @@ to be a self contained CDK app. The runner will execute the following for each f
 Example:
 
 ```bash
-integ-runner --update --parallel --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./
+integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./
 ```
 
 This will search for integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.
 
+If you are providing a list of tests to execute, either as CLI arguments or from a file, the name of the test needs to be relative to the `directory`.
+For example, if there is a test `aws-iam/test/integ.policy.js` and the current working directory is `aws-iam` you would provide `integ.policy.js`
+
+```bash
+yarn integ integ.policy.js
+```
+
 ### Common Workflow
 
 A common workflow to use when running integration tests is to first run the integration tests to see if there are any snapshot differences.

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -3,7 +3,7 @@ import * as path from 'path';
 import * as chalk from 'chalk';
 import * as workerpool from 'workerpool';
 import * as logger from './logger';
-import { IntegrationTests, IntegTestConfig } from './runner/integ-tests';
+import { IntegrationTests, IntegTestConfig } from './runner/integration-tests';
 import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
 
 // https://github.com/yargs/yargs/issues/1929
@@ -17,13 +17,12 @@ async function main() {
     .usage('Usage: integ-runner [TEST...]')
     .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' })
     .option('clean', { type: 'boolean', default: true, desc: 'Skips stack clean up after test is completed (use --no-clean to negate)' })
-    .option('verbose', { type: 'boolean', default: false, alias: 'v', desc: 'Verbose logs' })
+    .option('verbose', { type: 'boolean', default: false, alias: 'v', desc: 'Verbose logs and metrics on integration tests durations' })
     .option('dry-run', { type: 'boolean', default: false, desc: 'do not actually deploy the stack. just update the snapshot (not recommended!)' })
     .option('update-on-failed', { type: 'boolean', default: false, desc: 'rerun integration tests and update snapshots for failed tests.' })
     .option('force', { type: 'boolean', default: false, desc: 'Rerun all integration tests even if tests are passing' })
-    .option('parallel', { type: 'boolean', default: false, desc: 'run integration tests in parallel' })
-    .option('parallel-regions', { type: 'array', desc: 'if --parallel is used then these regions are used to run tests in parallel', nargs: 1, default: [] })
-    .options('directory', { type: 'string', default: 'test', desc: 'starting directory to discover integration tests' })
+    .option('parallel-regions', { type: 'array', desc: 'Tests are run in parallel across these regions. To prevent tests from running in parallel, provide only a single region', nargs: 1, default: [] })
+    .options('directory', { type: 'string', default: 'test', desc: 'starting directory to discover integration tests. Tests will be discovered recursively from this directory' })
     .options('profiles', { type: 'array', desc: 'list of AWS profiles to use. Tests will be run in parallel across each profile+regions', nargs: 1, default: [] })
     .options('max-workers', { type: 'number', desc: 'The max number of workerpool workers to use when running integration tests in parallel', default: 16 })
     .options('exclude', { type: 'boolean', desc: 'All tests should be run, except for the list of tests provided', default: false })
@@ -61,11 +60,11 @@ async function main() {
     if (argv._.length > 0 && fromFile) {
       throw new Error('A list of tests cannot be provided if "--from-file" is provided');
     } else if (argv._.length === 0 && !fromFile) {
-      testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromCliArgs()));
+      testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs()));
     } else if (fromFile) {
-      testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromFile(fromFile)));
+      testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromFile(path.resolve(fromFile))));
     } else {
-      testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromCliArgs(argv._.map((x: any) => x.toString()), exclude)));
+      testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs(argv._.map((x: any) => x.toString()), exclude)));
     }
 
     // always run snapshot tests, but if '--force' is passed then
@@ -93,7 +92,7 @@ async function main() {
         clean: argv.clean,
         dryRun: argv['dry-run'],
         verbose: argv.verbose,
-        updateWorkflow: !argv['disable-update-workflow'],
+        updateWorkflow: !!argv['disable-update-workflow'],
       });
 
       if (argv.clean === false) {

packages/@aws-cdk/integ-runner/lib/runner/index.ts

@@ -1,2 +1,5 @@
-export * from './runners';
-export * from './integ-tests';
+export * from './runner-base';
+export * from './integ-test-case';
+export * from './integ-test-runner';
+export * from './snapshot-test-runner';
+export * from './integration-tests';

packages/@aws-cdk/integ-runner/lib/runner/integ-test-case.ts

@@ -0,0 +1,230 @@
+import { TestCase, TestOptions } from '@aws-cdk/cloud-assembly-schema';
+import { ICdk, ListOptions } from 'cdk-cli-wrapper';
+import * as fs from 'fs-extra';
+import { IntegManifestReader } from './private/integ-manifest';
+
+const CDK_INTEG_STACK_PRAGMA = '/// !cdk-integ';
+const PRAGMA_PREFIX = 'pragma:';
+const SET_CONTEXT_PRAGMA_PREFIX = 'pragma:set-context:';
+const VERIFY_ASSET_HASHES = 'pragma:include-assets-hashes';
+const DISABLE_UPDATE_WORKFLOW = 'pragma:disable-update-workflow';
+const ENABLE_LOOKUPS_PRAGMA = 'pragma:enable-lookups';
+
+/**
+ * Represents an integration test
+ */
+export type TestCases = { [testName: string]: TestCase };
+
+/**
+ * Helper class for working with Integration tests
+ * This requires an `integ.json` file in the snapshot
+ * directory. For legacy test cases use LegacyIntegTestCases
+ */
+export class IntegTestCases {
+
+  /**
+   * Loads integ tests from a snapshot directory
+   */
+  public static fromPath(path: string): IntegTestCases {
+    const reader = IntegManifestReader.fromPath(path);
+    return new IntegTestCases(
+      reader.tests.enableLookups,
+      reader.tests.testCases,
+    );
+  }
+
+  constructor(
+    public readonly enableLookups: boolean,
+    public readonly testCases: TestCases,
+  ) {}
+
+  /**
+   * Returns a list of stacks that have stackUpdateWorkflow disabled
+   */
+  public getStacksWithoutUpdateWorkflow(): string[] {
+    const stacks: string[] = [];
+    for (const testCase of Object.values(this.testCases ?? {})) {
+      const update = testCase.stackUpdateWorkflow ?? true;
+      if (!update) {
+        stacks.push(...testCase.stacks);
+      }
+    }
+    return stacks;
+  }
+
+  /**
+   * Returns test case options for a given stack
+   */
+  public getOptionsForStack(stackId: string): TestOptions | undefined {
+    for (const testCase of Object.values(this.testCases ?? {})) {
+      if (testCase.stacks.includes(stackId)) {
+        return {
+          hooks: testCase.hooks,
+          regions: testCase.regions,
+          diffAssets: testCase.diffAssets ?? false,
+          allowDestroy: testCase.allowDestroy,
+          cdkCommandOptions: testCase.cdkCommandOptions,
+          stackUpdateWorkflow: testCase.stackUpdateWorkflow ?? true,
+        };
+      }
+    }
+    return undefined;
+  }
+}
+
+/**
+ * Options for a reading a legacy test case manifest
+ */
+export interface LegacyTestCaseConfig {
+  /**
+   * The name of the test case
+   */
+  readonly testName: string;
+
+  /**
+   * Options to use when performing `cdk list`
+   * This is used to determine the name of the stacks
+   * in the test case
+   */
+  readonly listOptions: ListOptions;
+
+  /**
+   * An instance of the CDK CLI (e.g. CdkCliWrapper)
+   */
+  readonly cdk: ICdk;
+
+  /**
+   * The path to the integration test file
+   * i.e. integ.test.js
+   */
+  readonly integSourceFilePath: string;
+}
+
+/**
+ * Helper class for creating an integ manifest for legacy
+ * test cases, i.e. tests without a `integ.json`.
+ */
+export class LegacyIntegTestCases extends IntegTestCases {
+
+  /**
+   * Returns the single test stack to use.
+   *
+   * If the test has a single stack, it will be chosen. Otherwise a pragma is expected within the
+   * test file the name of the stack:
+   *
+   * @example
+   *
+   *    /// !cdk-integ <stack-name>
+   *
+   */
+  public static fromLegacy(config: LegacyTestCaseConfig): LegacyIntegTestCases {
+    const pragmas = this.pragmas(config.integSourceFilePath);
+    const tests: TestCase = {
+      stacks: [],
+      diffAssets: pragmas.includes(VERIFY_ASSET_HASHES),
+      stackUpdateWorkflow: !pragmas.includes(DISABLE_UPDATE_WORKFLOW),
+    };
+    const pragma = this.readStackPragma(config.integSourceFilePath);
+    if (pragma.length > 0) {
+      tests.stacks.push(...pragma);
+    } else {
+      const stacks = (config.cdk.list({
+        ...config.listOptions,
+      })).split('\n');
+      if (stacks.length !== 1) {
+        throw new Error('"cdk-integ" can only operate on apps with a single stack.\n\n' +
+          '  If your app has multiple stacks, specify which stack to select by adding this to your test source:\n\n' +
+          `      ${CDK_INTEG_STACK_PRAGMA} STACK ...\n\n` +
+          `  Available stacks: ${stacks.join(' ')} (wildcards are also supported)\n`);
+      }
+      if (stacks.length === 1 && stacks[0] === '') {
+        throw new Error(`No stack found for test ${config.testName}`);
+      }
+      tests.stacks.push(...stacks);
+    }
+
+    return new LegacyIntegTestCases(
+      pragmas.includes(ENABLE_LOOKUPS_PRAGMA),
+      {
+        [config.testName]: tests,
+      },
+    );
+  }
+
+  public static getPragmaContext(integSourceFilePath: string): Record<string, any> {
+    const ctxPragmaContext: Record<string, any> = {};
+
+    // apply context from set-context pragma
+    // usage: pragma:set-context:key=value
+    const ctxPragmas = (this.pragmas(integSourceFilePath)).filter(p => p.startsWith(SET_CONTEXT_PRAGMA_PREFIX));
+    for (const p of ctxPragmas) {
+      const instruction = p.substring(SET_CONTEXT_PRAGMA_PREFIX.length);
+      const [key, value] = instruction.split('=');
+      if (key == null || value == null) {
+        throw new Error(`invalid "set-context" pragma syntax. example: "pragma:set-context:@aws-cdk/core:newStyleStackSynthesis=true" got: ${p}`);
+      }
+
+      ctxPragmaContext[key] = value;
+    }
+    return {
+      ...ctxPragmaContext,
+    };
+  }
+
+
+  /**
+   * Reads stack names from the "!cdk-integ" pragma.
+   *
+   * Every word that's NOT prefixed by "pragma:" is considered a stack name.
+   *
+   * @example
+   *
+   *    /// !cdk-integ <stack-name>
+   */
+  private static readStackPragma(integSourceFilePath: string): string[] {
+    return (this.readIntegPragma(integSourceFilePath)).filter(p => !p.startsWith(PRAGMA_PREFIX));
+  }
+
+  /**
+   * Read arbitrary cdk-integ pragma directives
+   *
+   * Reads the test source file and looks for the "!cdk-integ" pragma. If it exists, returns it's
+   * contents. This allows integ tests to supply custom command line arguments to "cdk deploy" and "cdk synth".
+   *
+   * @example
+   *
+   *    /// !cdk-integ [...]
+   */
+  private static readIntegPragma(integSourceFilePath: string): string[] {
+    const source = fs.readFileSync(integSourceFilePath, { encoding: 'utf-8' });
+    const pragmaLine = source.split('\n').find(x => x.startsWith(CDK_INTEG_STACK_PRAGMA + ' '));
+    if (!pragmaLine) {
+      return [];
+    }
+
+    const args = pragmaLine.substring(CDK_INTEG_STACK_PRAGMA.length).trim().split(' ');
+    if (args.length === 0) {
+      throw new Error(`Invalid syntax for cdk-integ pragma. Usage: "${CDK_INTEG_STACK_PRAGMA} [STACK] [pragma:PRAGMA] [...]"`);
+    }
+    return args;
+  }
+
+  /**
+   * Return the non-stack pragmas
+   *
+   * These are all pragmas that start with "pragma:".
+   *
+   * For backwards compatibility reasons, all pragmas that DON'T start with this
+   * string are considered to be stack names.
+   */
+  private static pragmas(integSourceFilePath: string): string[] {
+    return (this.readIntegPragma(integSourceFilePath)).filter(p => p.startsWith(PRAGMA_PREFIX));
+  }
+
+  constructor(
+    public readonly enableLookups: boolean,
+    public readonly testCases: TestCases,
+  ) {
+    super(enableLookups, testCases);
+  }
+}