add benchmark

improve logger

.github/workflows/benchmark.yml

@@ -0,0 +1,41 @@
+name: Benchmark
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Check performances
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Build
+        run: yarn build
+
+      - name: Run benchmark
+        run: yarn benchmark | tee output.txt
+
+      - name: Store benchmark result
+        uses: rhysd/github-action-benchmark@v1
+        with:
+          name: '"with ts-gql-plugin" vs "without ts-gql-plugin" Benchmark'
+          tool: 'benchmarkjs'
+          output-file-path: output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN  }}
+          auto-push: true
+          alert-threshold: '130%'
+          comment-always: true
+          comment-on-alert: true
+          fail-on-alert: true

README.md

@@ -87,7 +87,7 @@ Configuration can be done at 2 levels: in tsconfig.json and in graphql-config fi
 | logLevel          | Optional. Plugin log level. Values `'default'` - `'verbose'` - `'debug'`. Default `'default'`.                                                                                                        |
 | projectNameRegex  | Optional. For multi-projects GraphQL config, regex for extracting project name from operation.                                                                                                        |
 
-> Log level `'debug'` writes log files into `ts-gql-plugin-logs` directory.
+> Log level `'debug'` writes log files into `ts-gql-plugin-logs` directory. When running by VSCode this directory can be hard to find, checkout TSServer logs where files paths are logged.
 > These logs contain updated code with hidden types generated by plugin.
 
 > Checkout config type in [plugin-config.ts](./src/plugin-config.ts).

benchmark/benchmark.ts

@@ -0,0 +1,56 @@
+#!/usr/bin/env ts-node
+
+import Benchmark, { Suite } from 'benchmark';
+import { execSync } from 'node:child_process';
+
+/**
+ * Run benchmark comparing "with plugin" vs "without plugin".
+ * Log diff in %.
+ *
+ * `yarn build` should be done before this process.
+ */
+
+const runBenchmark = (): void => {
+  const suite = new Suite()
+    .add('compilation with plugin', () => {
+      execSync('yarn tsc-ls -b ./example/tsconfig.benchmark1.json --force');
+    })
+    .add('compilation without plugin', () => {
+      execSync('yarn tsc-ls -b ./example/tsconfig.benchmark2.json --force');
+    })
+    .run();
+
+  const benchmarks: Benchmark[] = suite.slice(0, suite.length);
+  const withPluginsBench = benchmarks[0];
+  const withoutPluginsBench = benchmarks[1];
+
+  const rmeMax = Math.max(
+    withPluginsBench.stats.rme,
+    withoutPluginsBench.stats.rme
+  );
+
+  // if uncertainty of measurement is >10%, re-run benchmark
+  if (rmeMax > 10) {
+    return runBenchmark();
+  }
+
+  /* eslint-disable no-mixed-operators */
+  const diffPercent = Math.max(
+    100 - (withPluginsBench.hz * 100) / withoutPluginsBench.hz,
+    0
+  );
+
+  const name =
+    'performance impact %: "with ts-gql-plugin" vs "without ts-gql-plugin"';
+  const value = diffPercent.toFixed(2);
+  const unit = '%';
+  const pm = '\u00B1';
+  const rme = rmeMax.toFixed(2);
+  const size =
+    withPluginsBench.stats.sample.length +
+    withoutPluginsBench.stats.sample.length;
+
+  console.log(`${name} x ${value} ${unit} ${pm}${rme}% (${size} runs sampled)`);
+};
+
+runBenchmark();

example/tsconfig.benchmark1.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "tsBuildInfoFile": "./node_modules/.cache/.tsbuildinfo",
+    "outDir": "./node_modules/.cache/dist",
+    "moduleResolution": "node",
+    "plugins": [
+      {
+        "name": "ts-gql-plugin",
+        "logLevel": "debug",
+        "projectNameRegex": "([A-Z][a-z]*)"
+      }
+    ]
+  },
+  "include": ["index.ts"]
+}

example/tsconfig.benchmark2.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "tsBuildInfoFile": "./node_modules/.cache/.tsbuildinfo",
+    "outDir": "./node_modules/.cache/dist",
+    "moduleResolution": "node"
+  },
+  "include": ["index.ts"]
+}

example/tsconfig.json

@@ -1,8 +1,10 @@
 {
   "extends": "../tsconfig.base.json",
   "compilerOptions": {
+    "composite": true,
+    "tsBuildInfoFile": "./node_modules/.cache/.tsbuildinfo",
+    "outDir": "./node_modules/.cache/dist",
     "moduleResolution": "node",
-    "noEmit": true,
     "plugins": [
       {
         "name": "ts-gql-plugin",

package.json

@@ -32,11 +32,11 @@
     "c:format": "prettier -w .",
     "c:lint": "eslint . --cache --cache-location node_modules/.cache/.eslintcache",
     "c:lint:fix": "yarn c:lint --fix",
-    "c:type": "tsc --noEmit",
+    "c:type": "tsc-ls -b",
     "c:test": "yarn c:test:watch --run",
     "c:test:watch": "vitest --outputTruncateLength=9999 --passWithNoTests",
     "build": "tsc -p tsconfig.build.json",
-    "c:type-gql": "tsc-ls -b ./example"
+    "benchmark": "ts-node --transpile-only benchmark/benchmark.ts"
   },
   "dependencies": {
     "@graphql-codegen/core": "2.5.1",
@@ -54,11 +54,13 @@
     "@apollo/client": "3.6.7",
     "@babel/core": "7.17.9",
     "@babel/eslint-parser": "7.17.0",
+    "@types/benchmark": "2.1.1",
     "@types/deasync": "0.1.2",
     "@types/node": "17.0.23",
     "@types/prettier": "2.6.3",
     "@typescript-eslint/eslint-plugin": "5.18.0",
     "@typescript-eslint/parser": "5.18.0",
+    "benchmark": "2.1.4",
     "c8": "7.11.3",
     "eslint": "8.13.0",
     "eslint-config-prettier": "8.5.0",

src/plugin.ts

@@ -55,7 +55,7 @@ const init: PluginInit = ({ typescript: ts }) => ({
       (initialFn) =>
         (fileName, scriptSnapshot, ...rest) => {
           if (isValidFilename(fileName)) {
-            logger.verbose(`create - Filename ${fileName}`);
+            logger.verbose(() => `create - Filename ${fileName}`);
             const debugTime = logger.debugTime();
 
             resetFileDiagnostics(fileName);
@@ -84,7 +84,7 @@ const init: PluginInit = ({ typescript: ts }) => ({
       (initialFn) =>
         (sourceFile, scriptSnapshot, ...rest) => {
           if (isValidSourceFile(sourceFile)) {
-            logger.verbose(`update - Filename ${sourceFile.fileName}`);
+            logger.verbose(() => `update - Filename ${sourceFile.fileName}`);
             const debugTime = logger.debugTime();
 
             resetFileDiagnostics(sourceFile.fileName);

src/source-update/create-source-updater.test.ts

@@ -18,6 +18,8 @@ describe('Create source updater', () => {
     error: vi.fn(),
     verbose: vi.fn(),
     debug: vi.fn(),
+    debugTime: vi.fn(),
+    debugToFile: vi.fn(),
     setFilename: vi.fn(),
   });
 

src/utils/logger.ts

@@ -26,19 +26,19 @@ export const createLogger = (
   return {
     log,
     error,
-    verbose: (message: string) => {
+    verbose: (message: () => string) => {
       if (!verbose && !debug) {
         return;
       }
 
-      log(message);
+      log(message());
     },
-    debug: (message: string) => {
+    debug: (message: () => string) => {
       if (!debug) {
         return;
       }
 
-      log(message);
+      log(message());
     },
     debugTime: () => {
       if (!debug) {
@@ -64,7 +64,11 @@ export const createLogger = (
 
       fs.mkdirSync(dir, { recursive: true });
 
-      fs.writeFileSync(path.join(dir, logFileName), message());
+      const filePath = path.join(process.cwd(), dir, logFileName);
+
+      fs.writeFileSync(filePath, message());
+
+      log(`Debug log written to ${filePath}`);
     },
     setFilename: (newFilename: string) => {
       filename = newFilename;

tsconfig.json

@@ -3,7 +3,13 @@
   "compilerOptions": {
     "incremental": true,
     "types": ["vitest/globals"],
-    "tsBuildInfoFile": "./node_modules/.cache/.tsbuildinfo"
+    "tsBuildInfoFile": "./node_modules/.cache/.tsbuildinfo",
+    "outDir": "./node_modules/.cache/dist"
   },
-  "include": ["src", "./*.js", "./*.ts"]
+  "include": ["src", "benchmark", "./*.js", "./*.ts"],
+  "references": [
+    {
+      "path": "example"
+    }
+  ]
 }

yarn.lock

@@ -1257,6 +1257,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/benchmark@npm:2.1.1":
+  version: 2.1.1
+  resolution: "@types/benchmark@npm:2.1.1"
+  checksum: e5b1cea7b9ee066783ac34513b88a67016342f89c015274eadbe851e034734ba86749da805895041ae722099f9b2ab46384aebd70a416a0623a599bcc71fd03a
+  languageName: node
+  linkType: hard
+
 "@types/chai-subset@npm:^1.3.3":
   version: 1.3.3
   resolution: "@types/chai-subset@npm:1.3.3"
@@ -1748,6 +1755,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"benchmark@npm:2.1.4":
+  version: 2.1.4
+  resolution: "benchmark@npm:2.1.4"
+  dependencies:
+    lodash: ^4.17.4
+    platform: ^1.3.3
+  checksum: aa466561d4f2b0a2419a3069b8f90fd35ffacf26849697eea9de525ecfbd10b44da11070cc51c88d772076db8cb2415641b493de7d6c024fdf8551019c6fcf1c
+  languageName: node
+  linkType: hard
+
 "bindings@npm:^1.5.0":
   version: 1.5.0
   resolution: "bindings@npm:1.5.0"
@@ -4043,7 +4060,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash@npm:^4.17.21, lodash@npm:~4.17.0":
+"lodash@npm:^4.17.21, lodash@npm:^4.17.4, lodash@npm:~4.17.0":
   version: 4.17.21
   resolution: "lodash@npm:4.17.21"
   checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7
@@ -4764,6 +4781,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"platform@npm:^1.3.3":
+  version: 1.3.6
+  resolution: "platform@npm:1.3.6"
+  checksum: 6f472a09c61d418c7e26c1c16d0bdc029549d512dbec6526216a1e59ec68100d07007d0097dcba69dddad883d6f2a83361b4bdfe0094a3d9a2af24158643d85e
+  languageName: node
+  linkType: hard
+
 "pluralize@npm:^8.0.0":
   version: 8.0.0
   resolution: "pluralize@npm:8.0.0"
@@ -5561,11 +5585,13 @@ __metadata:
     "@graphql-codegen/core": 2.5.1
     "@graphql-codegen/typescript": 2.4.10
     "@graphql-codegen/typescript-operations": 2.3.7
+    "@types/benchmark": 2.1.1
     "@types/deasync": 0.1.2
     "@types/node": 17.0.23
     "@types/prettier": 2.6.3
     "@typescript-eslint/eslint-plugin": 5.18.0
     "@typescript-eslint/parser": 5.18.0
+    benchmark: 2.1.4
     c8: 7.11.3
     deasync: 0.1.26
     eslint: 8.13.0