Merge pull request #2058 from BetterThanTomorrow/pez/e2e-tests-config…

…urability

Make e2e test runner re-usable

.circleci/config.yml

@@ -218,7 +218,7 @@ jobs:
           command: npm run integration-test
       - store_test_results:
           path: ~/calva/junit
-  test-vsix:
+  test-e2e:
     docker:
       - image: cimg/clojure:1.11-browsers
     working_directory: ~/calva
@@ -233,7 +233,25 @@ jobs:
           command: sudo apt update && sudo apt install -y libnss3
       - run:
           name: Run VSIX E2E Tests
-          command: npm run vsix-test /tmp/artifacts/calva-$(node -p 'require("./package.json").version').vsix
+          command: npm run e2e-test -- --calva-vsix=/tmp/artifacts/calva-$(node -p 'require("./package.json").version').vsix --test-workspace=src/extension-test/e2e-test
+      - store_test_results:
+          path: ~/calva/junit
+  test-e2e-sub-projects:
+    docker:
+      - image: cimg/clojure:1.11-browsers
+    working_directory: ~/calva
+    steps:
+      - attach_workspace:
+          at: /tmp
+      - run:
+          name: Restore build
+          command: rmdir ~/calva && cp -r /tmp/build ~/calva
+      - run:
+          name: Apt install missing dependencies
+          command: sudo apt update && sudo apt install -y libnss3
+      - run:
+          name: Run VSIX E2E Tests for test-data/projects/e2e-sub-projects
+          command: npm run e2e-test -- --calva-vsix=/tmp/artifacts/calva-$(node -p 'require("./package.json").version').vsix --test-workspace=test-data/projects/e2e-sub-projects
       - store_test_results:
           path: ~/calva/junit
   github-release:
@@ -446,7 +464,10 @@ workflows:
       - test-integration:
           requires:
             - build
-      - test-vsix:
+      - test-e2e:
+          requires:
+            - build
+      - test-e2e-sub-projects:
           requires:
             - build
       - test-ts-unit:
@@ -459,7 +480,8 @@ workflows:
             - test-grammar
             - test-cljslib
             - test-integration
-            - test-vsix
+            - test-e2e
+            - test-e2e-sub-projects
             - test-ts-unit
           filters:
             branches:
@@ -521,7 +543,15 @@ workflows:
               ignore: /.*/
             tags:
               only: /^v\d+\.\d+\.\d+-?.*/
-      - test-vsix:
+      - test-e2e:
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v\d+\.\d+\.\d+-?.*/
+      - test-e2e-sub-projects:
           requires:
             - build
           filters:
@@ -544,7 +574,8 @@ workflows:
             - test-grammar
             - test-cljslib
             - test-integration
-            - test-vsix
+            - test-e2e
+            - test-e2e-sub-projects
             - test-ts-unit
           filters:
             branches:

CHANGELOG.md

@@ -4,6 +4,8 @@ Changes to Calva.
 
 ## [Unreleased]
 
+- [Make e2e test runner re-usable](https://github.com/BetterThanTomorrow/calva/issues/2058)
+
 ## [2.0.329] - 2023-02-01
 
 - [Sort aliases for deps.edn projects](https://github.com/BetterThanTomorrow/calva/issues/2035)

package.json

@@ -2889,7 +2889,8 @@
     "bump-version": "npm set git-tag-version false && npm version patch",
     "calva-lib-test": "node ./out/cljs-lib/test/cljs-lib-tests.js",
     "integration-test": "node ./out/extension-test/integration/runTests.js",
-    "vsix-test": "node ./vsix-test/launch.js",
+    "e2e-test": "node ./src/extension-test/e2e-test/launch.js",
+    "pree2e-test": "cd ./src/extension-test/e2e-test/ && npm i",
     "unit-test": "npx mocha --require ts-node/register 'src/extension-test/unit/**/*-test.ts'",
     "unit-test-watch": "npx mocha --watch --require ts-node/register --watch-extensions ts --watch-files src 'src/extension-test/unit/**/*-test.ts'",
     "prettier-format": "npx prettier --write \"./**/*.{ts,js,json}\"",

src/extension-test/e2e-test/.joyride/src/config.cljs

@@ -0,0 +1,4 @@
+(ns config)
+
+(defn ns-symbols []
+  ['tests.a-test])

vsix-test/.joyride/src/test_runner.cljs → src/extension-test/e2e-test/.joyride/src/test_runner.cljs

@@ -1,6 +1,7 @@
 (ns test-runner
   (:require [clojure.string :as string]
             [cljs.test]
+            [config :as config]
             [db :as db]
             [promesa.core :as p]))
 
@@ -16,7 +17,7 @@
 (def old-pass (get-method cljs.test/report [:cljs.test/default :pass]))
 
 (defmethod cljs.test/report [:cljs.test/default :pass] [m]
-  (binding [*print-fn* write] (old-pass m)) 
+  (binding [*print-fn* write] (old-pass m))
   (write "✅")
   (swap! db/!state update :pass inc))
 
@@ -55,16 +56,19 @@
   (if (:ws-activated? @db/!state)
     (do
       (println "Runner: Workspace activated, running tests...")
-      (require '[tests.a-test])
-      (cljs.test/run-tests 'tests.a-test))
+      (try
+        (doseq [ns-sym (config/ns-symbols)]
+          (require ns-sym)
+          (cljs.test/run-tests ns-sym))
+        (catch :default e
+          (p/reject! (:running @db/!state) e))))
     (do
       (println "Runner: Workspace not activated yet, tries: " tries "- trying again in a jiffy")
       (js/setTimeout #(run-when-ws-activated (inc tries)) 10))))
 
 (defn run-all-tests []
   (let [running (p/deferred)]
-    (swap! db/!state assoc
-           :running running)
+    (swap! db/!state assoc :running running)
     (run-when-ws-activated 1)
     running))
 

vsix-test/launch.js → src/extension-test/e2e-test/launch.js

@@ -29,41 +29,43 @@ function init() {
   });
 }
 
-async function main(calvaVSIXPath) {
+async function main(calvaVSIXPathOrLabel, testWorkspace) {
   try {
     const extensionTestsPath = path.resolve(__dirname, 'runTests');
     const vscodeExecutablePath = await downloadAndUnzipVSCode('insiders');
     const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath);
-    const testWorkspace = path.resolve(__dirname);
 
     const launchArgs = [
       testWorkspace,
       '--disable-workspace-trust',
-      '--install-extension',
-      // TODO: Build the extension first, and use the built version here
-      //       In CI we will have built the extension already, so it is
-      //       more a TODO to figure out where to find the built extension
-      calvaVSIXPath,
-      // WHen debugging tests, it can be good to use the development version of Joyride
+      // When debugging tests, it can be good to use the development version of Joyride
       // If you do, comment out the install of the Joyride extension here
       // And set the `extensionDevelopmentPath` in the `runTests` call below
+      // (And can't be used if you are testing the development version of Calva)
       '--install-extension',
       'betterthantomorrow.joyride',
     ];
+    if (calvaVSIXPathOrLabel !== 'extension-development') {
+      launchArgs.push('--install-extension', calvaVSIXPathOrLabel);
+    }
 
     cp.spawnSync(cliPath, launchArgs, {
       encoding: 'utf-8',
       stdio: 'inherit',
     });
 
-    await runTests({
+    const runOptions = {
       // When debugging tests, it can be good to use the development version Joyride
       // extensionDevelopmentPath: '/Users/pez/Projects/joyride',
       vscodeExecutablePath,
       reuseMachineInstall: true,
       extensionTestsPath,
       launchArgs: [testWorkspace],
-    })
+    };
+    if (calvaVSIXPathOrLabel === 'extension-development') {
+      runOptions.extensionDevelopmentPath = path.resolve(__dirname, '..');
+    }
+    await runTests(runOptions)
       .then((_result) => {
         console.info('Tests finished');
       })
@@ -77,14 +79,15 @@ async function main(calvaVSIXPath) {
   }
 }
 
-const calvaVSIXPath = process.argv[2];
-if (!calvaVSIXPath) {
-  console.error('Missing path to Calva VSIX file');
-  process.exit(1);
-}
+const args = require('minimist')(process.argv.slice(2));
+const calvaVSIX = args['calva-vsix'] ? args['calva-vsix'] : 'extension-development';
+const testWorkspace = args['test-workspace']
+  ? path.resolve(args['test-workspace'])
+  : path.resolve(__dirname);
+console.info(`Using:\n  Calva: ${calvaVSIX}\n  Test workspace: ${testWorkspace}`);
 
 void init()
-  .then(() => main(calvaVSIXPath))
+  .then(() => main(calvaVSIX, testWorkspace))
   .catch((error) => {
     console.error('Failed to initialize test running environment:', error);
     process.exit(1);

vsix-test/package.json → src/extension-test/e2e-test/package.json

@@ -9,5 +9,7 @@
     "email": "pez@pezius.com"
   },
   "license": "MIT",
-  "dependencies": {}
+  "dependencies": {
+    "minimist": "^1.2.7"
+  }
 }

test-data/projects/e2e-sub-projects/.joyride/scripts/workspace_activate.cljs

@@ -0,0 +1 @@
+../../../../../src/extension-test/e2e-test/.joyride/scripts/workspace_activate.cljs

test-data/projects/e2e-sub-projects/.joyride/src/config.cljs

@@ -0,0 +1,4 @@
+(ns config)
+
+(defn ns-symbols []
+  ['tests.example-test])

test-data/projects/e2e-sub-projects/.joyride/src/db.cljs

@@ -0,0 +1 @@
+../../../../../src/extension-test/e2e-test/.joyride/src/db.cljs

test-data/projects/e2e-sub-projects/.joyride/src/macros.cljs

@@ -0,0 +1 @@
+../../../../../src/extension-test/e2e-test/.joyride/src/macros.cljs

test-data/projects/e2e-sub-projects/.joyride/src/test_runner.cljs

@@ -0,0 +1 @@
+../../../../../src/extension-test/e2e-test/.joyride/src/test_runner.cljs

test-data/projects/e2e-sub-projects/.joyride/src/tests/example_test.cljs

@@ -0,0 +1,13 @@
+(ns tests.example-test
+  (:require [cljs.test :refer [deftest testing is]]
+            [promesa.core :as p]
+            [macros :refer [deftest-async]]))
+
+(deftest a-sync-test
+  (testing "A sync test"
+    (is (not= nil? 1))))
+
+(deftest-async an-async-test
+  (testing "Requires the Joyride extension"
+    (p/let [answer (p/resolved 42)]
+      (is (= 42 answer)))))

test-data/projects/e2e-sub-projects/no-project-here/project-a/a.cljs

@@ -0,0 +1 @@
+(ns a)

test-data/projects/e2e-sub-projects/no-project-here/project-b/b.cljs

@@ -0,0 +1 @@
+(ns b)

test-data/projects/e2e-sub-projects/project-a/a.cljs

@@ -0,0 +1 @@
+(ns a)

test-data/projects/e2e-sub-projects/project-a/deps.edn

@@ -0,0 +1 @@
+{:paths ["."]}

test-data/projects/e2e-sub-projects/project-b/b.cljs

@@ -0,0 +1 @@
+(ns b)

test-data/projects/e2e-sub-projects/sub-projects/proj1/deps.edn

@@ -0,0 +1 @@
+{}