feat(typescript-checker): a typescript type checker plugin (#2241)

Introducing the typescript type-checker plugin. 

## Features

👽 Type check each mutant. Invalid mutants will be marked as `CompileError` in your Stryker report.  
🧒 Easy to setup, only your `tsconfig.json` file is needed.  
🔢 Type check is done in-memory, no side effects on disk.  
🎁 Support for both single typescript projects as well as projects with project references (`--build` mode).  

## Configuring

You can configure the typescript checker in the `stryker.conf.js` (or `stryker.conf.json`) file.

```js
// stryker.conf.json
{
  "checkers": ["typescript"],
  "typescriptChecker": {
    "tsconfigFile": "tsconfig.json"
  }
}
```

### `typescriptChecker.tsconfigFile` [`string`]

Default: `'tsconfig.json'`

The path to your [tsconfig](https://www.typescriptlang.org/docs/handbook/tsconfig-json.html). Project references _are supported_, `--build` mode will be enabled automatically when references are found in your tsconfig.json file.

## Implemenation

The `@stryker-mutator/typescript-checker` is not using the [typescript language service api](https://github.com/Microsoft/TypeScript/wiki/Using-the-Language-Service-API), like the current [typescript transpiler](https://github.com/stryker-mutator/stryker/blob/f44008993a543dc3f38ca99516f56d315fdcb735/packages/typescript/src/transpiler/TranspilingLanguageService.ts#L23) is. Instead, it's using the [compiler API](https://github.com/microsoft/TypeScript/wiki/Using-the-Compiler-API). The language service API wasn't really designed for our use case. For example, it doesn't support project references, nor will it ever (read this somewhere, unfortunately, cannot find the source). It also took a lot of code to get working, with a pretty high number of bugs 🐛. Even in our own project, we needed separate `tsconfig.stryker.json` config files. Very annoying.

The compiler API is somewhat more low-level. We're starting the compiler on `initialize()` method of the checker API. If the initial build fails, we're rejecting the promise. Each mutant is checked basically the same way as it was in the old typescript transpiler. Namely, the previous mutant is removed, the new mutant is placed and the typescript compiler runs. If it errors, the `CheckStatus.CompilError` status is reported with the error text in the `reason` field.

There are some nifty tricks to be found:

1. During `initialize()` the typescript compiler starts. We're reading all files from disk synchronously and storing them in memory. After the initial build, all files should be in-memory, no further files are pulled in.
    * We're using the `ts.createSolutionBuilderWithWatch`. This creates a compiler with `--build` mode enabled. This seems to work for both single-project solutions as for a multi project solution with project references. This api is available since ts `3.6`, so we're throwing an error if a previous version of typescript is found.
1. The way to inform the typescript compiler that a file has changed is to "pretend" as if the file changed on disk. The file system watcher is informed in-memory of the change. This triggers a rebuild.
1. In order to still override compiler options as specified in #391, we're transforming each tsconfig.json file as it is read from disk. This way we can force the compiler options we want. The config file can also be named differently, each tsconfig file is scanned for `references` in order to make sure we're only transforming the tsconfig files that are used.
1. In order to ensure the best performance, `--noEmit` is forced. This will prevent the typescript compiler from outputting code. Unfortunately  `--noEmit` is forbidden for `--build` mode, which is why `--emitDeclarationsOnly` is forced for those projects.

.vscode/launch.json

@@ -4,6 +4,15 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Program",
+      "program": "${file}",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    },
     {
       "type": "node",
       "request": "launch",
@@ -60,12 +69,6 @@
       ],
       "internalConsoleOptions": "openOnSessionStart"
     },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Launch Program",
-      "program": "${workspaceRoot}\\packages\\stryker-mocha-runner\""
-    },
     {
       "type": "node",
       "request": "attach",

packages/api/src/check/MutantStatus.ts

@@ -0,0 +1,10 @@
+export enum MutantStatus {
+  Init = 'init',
+  Ignored = 'ignored',
+  NoCoverage = 'noCoverage',
+  Killed = 'killed',
+  Survived = 'survived',
+  TimedOut = 'timedOut',
+  RuntimeError = 'runtimeError',
+  CompileError = 'compileError',
+}

packages/typescript-checker/.eslintrc

@@ -0,0 +1,3 @@
+{
+    "extends": "../../.eslintrc.js"
+}

packages/typescript-checker/.mocharc.jsonc

@@ -0,0 +1,12 @@
+{
+  "require": ["source-map-support/register"],
+  "file": [
+    // This file is included first
+    "dist/test/helpers/setup-tests.js"
+  ],
+  "spec": [
+    "dist/test/unit/**/*.js",
+    "dist/test/integration/**/*.js"
+  ],
+  "timeout": 10000
+}

packages/typescript-checker/.npmignore

@@ -0,0 +1,6 @@
+*
+
+!dist/src/**/*.js
+!dist/src/**/*.map
+!dist/src/**/*.ts
+!src/**/*

packages/typescript-checker/.npmrc

@@ -0,0 +1 @@
+package-lock=false

packages/typescript-checker/.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Unit / Integration tests",
+      "program": "${workspaceRoot}/../../node_modules/mocha/bin/_mocha",
+      "internalConsoleOptions": "openOnSessionStart",
+      "outFiles": [
+        "${workspaceRoot}/dist/**/*.js"
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "args": [
+        "--no-timeout"
+      ]
+    }
+  ]
+}

packages/typescript-checker/README.md

@@ -0,0 +1,84 @@
+**NOTE:** This readme describes a checker plugin set to release in Stryker v4. It is not usable yet! 
+Please see the [master branch](https://github.com/stryker-mutator/stryker/tree/master) for the current release of Stryker. 
+
+
+[![Mutation testing badge](https://img.shields.io/endpoint?style=flat&url=https%3A%2F%2Fbadge-api.stryker-mutator.io%2Fgithub.com%2Fstryker-mutator%2Fstryker%2Fmaster%3Fmodule%3Dtypescript)](https://dashboard.stryker-mutator.io/reports/github.com/stryker-mutator/stryker/master?module=typescript-checker)
+[![Build Status](https://github.com/stryker-mutator/stryker/workflows/CI/badge.svg)](https://github.com/stryker-mutator/stryker/actions?query=workflow%3ACI+branch%3Amaster)
+[![NPM](https://img.shields.io/npm/dm/@stryker-mutator/typescript-checker.svg)](https://www.npmjs.com/package/@stryker-mutator/typescript-checker)
+[![Node version](https://img.shields.io/node/v/@stryker-mutator/typescript.svg)](https://img.shields.io/node/v/@stryker-mutator/typescript-checker.svg)
+[![Slack Chat](https://img.shields.io/badge/slack-chat-brightgreen.svg?logo=slack)](https://join.slack.com/t/stryker-mutator/shared_invite/enQtOTUyMTYyNTg1NDQ0LTU4ODNmZDlmN2I3MmEyMTVhYjZlYmJkOThlNTY3NTM1M2QxYmM5YTM3ODQxYmJjY2YyYzllM2RkMmM1NjNjZjM)
+
+![Stryker](https://github.com/stryker-mutator/stryker/raw/master/stryker-80x80.png)
+
+# Typescript checker
+
+A TypeScript type checker plugin for [Stryker](https://stryker-mutator.io), the ~~JavaScript~~ *TypeScript* Mutation testing framework.
+This plugin enables type checking on mutants, so you won't have to waste time on mutants which results in a type error.
+
+## Features
+
+👽 Type check each mutant. Invalid mutants will be marked as `CompileError` in your Stryker report.  
+🧒 Easy to setup, only your `tsconfig.json` file is needed.  
+🔢 Type check is done in-memory, no side effects on disk.  
+🎁 Support for both single typescript projects as well as projects with project references (`--build` mode).  
+
+## Install
+
+First, install Stryker itself (you can follow the [quickstart on the website](https://stryker-mutator.io/quickstart.html))
+
+Next, install this package:
+
+```bash
+npm install --save-dev @stryker-mutator/typescript-checker
+```
+
+## Configuring
+
+You can configure the typescript checker in the `stryker.conf.js` (or `stryker.conf.json`) file.
+
+```js
+// stryker.conf.json
+{
+  "checkers": ["typescript"],
+  "typescriptChecker": {
+    "tsconfigFile": "tsconfig.json"
+  }
+}
+```
+
+### `typescriptChecker.tsconfigFile` [`string`]
+
+Default: `'tsconfig.json'`
+
+The path to your [tsconfig](https://www.typescriptlang.org/docs/handbook/tsconfig-json.html). Project references _are supported_, `--build` mode will be enabled automatically when references are found in your tsconfig.json file.
+
+*Note: the following compiler options are always overridden by @stryker-mutator/typescript-checker to ovoid false positives. See [issue 391](https://github.com/stryker-mutator/stryker/issues/391#issue-259829320) for more information on this*
+
+```json
+{
+  "compilerOptions": {
+    "allowUnreachableCode": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false
+  }
+}
+```
+
+## Peer dependencies
+
+The `@stryker-mutator/typescript-checker` package  for `stryker` to enable `typescript` support. As such, you should make sure you have the correct versions of its dependencies installed:
+
+* `typescript`
+* `@stryker-mutator/core`
+
+For the current versions, see the `peerDependencies` section in the [package.json](https://github.com/stryker-mutator/stryker/blob/master/packages/typescript/package.json).
+
+
+## Load the plugin
+
+In this plugin the `@stryker-mutator/typescript-checker`' must be loaded into Stryker.
+The easiest way to achieve this, is *not have a `plugins` section* in your config file. That way, all plugins starting with `"@stryker-mutator/"` will be loaded.
+
+If you do decide to choose specific modules, don't forget to add `"@stryker-mutator/typescript-checker"` to the list of plugins to load.
+
+

packages/typescript-checker/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@stryker-mutator/typescript-checker",
+  "version": "3.2.2",
+  "description": "A typescript type checker plugin to be used in Stryker, the JavaScript mutation testing framework",
+  "main": "dist/src/index.js",
+  "scripts": {
+    "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 80 --functions 80 --branches 75 npm run mocha",
+    "mocha": "mocha",
+    "stryker": "node ../core/bin/stryker run"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/stryker-mutator/stryker",
+    "directory": "packages/typescript-checker"
+  },
+  "engines": {
+    "node": ">=10"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "author": "Nico Jansen <jansennico@gmail.com>",
+  "contributors": [
+    "Simon de Lang <simondelang@gmail.com>",
+    "Nico Jansen <jansennico@gmail.com>"
+  ],
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/stryker-mutator/stryker/issues"
+  },
+  "homepage": "https://stryker-mutator.io",
+  "dependencies": {
+    "@stryker-mutator/api": "^3.2.2",
+    "@stryker-mutator/util": "^3.2.2",
+    "semver": "^7.3.2"
+  },
+  "devDependencies": {
+    "@stryker-mutator/test-helpers": "^3.2.2",
+    "@types/semver": "^7.2.0"
+  },
+  "peerDependencies": {
+    "typescript": ">=3.6",
+    "@stryker-mutator/core": "3.2.4"
+  }
+}

packages/typescript-checker/schema/typescript-checker-options.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "type": "object",
+  "title": "TypescriptCheckerOptions",
+  "additionalProperties": false,
+  "properties": {
+    "typescriptChecker": {
+      "title": "TypescriptOptions",
+      "additionalProperties": false,
+      "description": "Configuration for @stryker-mutator/typescript-checker.",
+      "default": {},
+      "type": "object",
+      "properties": {
+        "tsconfigFile": {
+          "description": "Location of your (root) tsconfig.json file. Project references with `--build` are supported by default.",
+          "type": "string",
+          "default": "tsconfig.json"
+        }
+      }
+    }
+  }
+}

packages/typescript-checker/src/fs/hybrid-file-system.ts

@@ -0,0 +1,69 @@
+import ts from 'typescript';
+import { Mutant } from '@stryker-mutator/api/core';
+
+import { ScriptFile } from './script-file';
+
+function toTSFileName(fileName: string) {
+  return fileName.replace(/\\/g, '/');
+}
+
+/**
+ * A very simple hybrid file system.
+ * * Readonly from disk
+ * * Writes in-memory
+ * * Hard caching
+ * * Ability to mutate one file
+ */
+export class HybridFileSystem {
+  private readonly files = new Map<string, ScriptFile | undefined>();
+  private mutatedFile: ScriptFile | undefined;
+
+  public writeFile(fileName: string, data: string) {
+    fileName = toTSFileName(fileName);
+    const existingFile = this.files.get(fileName);
+    if (existingFile) {
+      existingFile.write(data);
+    } else {
+      this.files.set(fileName, new ScriptFile(data, fileName));
+    }
+  }
+
+  public mutate(mutant: Pick<Mutant, 'fileName' | 'range' | 'replacement'>) {
+    const fileName = toTSFileName(mutant.fileName);
+    const file = this.files.get(fileName);
+    if (!file) {
+      throw new Error(`File "${mutant.fileName}" cannot be found.`);
+    }
+    if (this.mutatedFile && this.mutatedFile !== file) {
+      this.mutatedFile.resetMutant();
+    }
+    file.mutate(mutant);
+    this.mutatedFile = file;
+  }
+
+  public watchFile(fileName: string, watcher: ts.FileWatcherCallback) {
+    const file = this.getFile(fileName);
+    if (!file) {
+      throw new Error(`Cannot find file ${fileName} for watching`);
+    }
+    file.watcher = watcher;
+  }
+
+  public getFile(fileName: string): ScriptFile | undefined {
+    fileName = toTSFileName(fileName);
+    if (!this.files.has(fileName)) {
+      let content = ts.sys.readFile(fileName);
+      if (content) {
+        let modifiedTime = ts.sys.getModifiedTime!(fileName)!;
+        this.files.set(fileName, new ScriptFile(content, fileName, modifiedTime));
+      } else {
+        this.files.set(fileName, undefined);
+      }
+    }
+    return this.files.get(fileName);
+  }
+
+  public existsInMemory(fileName: string): boolean {
+    return !!this.files.get(toTSFileName(fileName));
+  }
+}

packages/typescript-checker/src/fs/index.ts

@@ -0,0 +1 @@
+export * from './hybrid-file-system';

packages/typescript-checker/src/fs/script-file.ts

@@ -0,0 +1,40 @@
+import ts from 'typescript';
+import { Mutant } from '@stryker-mutator/api/core';
+import { FileWatcherCallback } from 'typescript';
+
+export class ScriptFile {
+  private readonly originalContent: string;
+  constructor(public content: string, public fileName: string, public modifiedTime = new Date()) {
+    this.originalContent = content;
+  }
+
+  public write(content: string) {
+    this.content = content;
+    this.touch();
+  }
+
+  public watcher: FileWatcherCallback | undefined;
+
+  public mutate(mutant: Pick<Mutant, 'replacement' | 'range'>) {
+    this.guardMutationIsWatched();
+    this.content = `${this.originalContent.substr(0, mutant.range[0])}${mutant.replacement}${this.originalContent.substr(mutant.range[1])}`;
+    this.touch();
+  }
+
+  public resetMutant() {
+    this.guardMutationIsWatched();
+    this.content = this.originalContent;
+    this.touch();
+  }
+
+  private guardMutationIsWatched() {
+    if (!this.watcher) {
+      throw new Error(`No watcher registered for ${this.fileName}. Changes would go unnoticed`);
+    }
+  }
+
+  private touch() {
+    this.modifiedTime = new Date();
+    this.watcher?.(this.fileName, ts.FileWatcherEventKind.Changed);
+  }
+}

packages/typescript-checker/src/index.ts

@@ -0,0 +1,9 @@
+import { PluginKind, declareClassPlugin } from '@stryker-mutator/api/plugin';
+
+import strykerValidationSchema from '../schema/typescript-checker-options.json';
+
+import { TypescriptChecker } from './typescript-checker';
+
+export const strykerPlugins = [declareClassPlugin(PluginKind.Checker, 'typescript', TypescriptChecker)];
+
+export { strykerValidationSchema, TypescriptChecker };