Replace local tests with E2E tests (#60)

* Rename fake-project to e2e-tests

* Add e2e test runner

* Add default rollup e2e test and run e2e in npm test

* Add better logging for e2e tests

* Only run test & default-webpack when running npm test

* Throw error if can't find e2e package.json

* Add more E2E tests

* Remove rollup E2E tests

* Add E2E test filtering

* Add custom webpack E2E test

* Add note about separate babel-loader for coverage

* Run E2E tests using local directories

Can rely on --preserve-symlinks-main flag when things get complicated (i.e. uninstalling karmatic's devDep on webpack)

* Fix webpack-loader e2e test to display code-coverage

* Run npm installs serially first, then run tests serially after

* Add other tests to default e2e test

* Rename e2e tests to begin with webpack-

* Remove webpack devDep and remove local tests

* Add some function comments

* Ensure script exits with failure code if tests fail

* Fix e2e test package.json names

* Run test command separatedly

So task fails if test fail

* Ensure karmatic exits with error code if error is thrown

* Use NODE_PRESERVE_SYMLINKS flag to fix E2E tests

* Only use preserve symlinks flag when running tests, not when installing

* Add comment explaining use of symlink env vars

* Add back test:build and test:watch scripts

* Remove `npm i` from `test:build` since it is redundant with `prepare` script

.github/workflows/main.yml

@@ -12,9 +12,8 @@ jobs:
         with:
           node-version: 12
       - name: npm install, build, and test
+        # npm install runs build and test as well
         run: |
           npm install
-          npm run build --if-present
-          npm test
         env:
           CI: true

e2e-test/webpack-custom/package.json

@@ -0,0 +1,12 @@
+{
+	"name": "karmatic-e2e-webpack-custom",
+	"description": "Test custom webpack config in karmatic using a custom Babel config",
+	"private": true,
+	"dependencies": {
+		"@babel/core": "^7.10.3",
+		"babel-loader": "8.1.0",
+		"babel-plugin-transform-rename-properties": "^0.1.0",
+		"karmatic": "file:../..",
+		"webpack": "^4.44.1"
+	}
+}

e2e-test/webpack-custom/src/index.js

@@ -0,0 +1,3 @@
+export function box(value) {
+	return { _value: value };
+}

e2e-test/webpack-custom/test/index.test.js

@@ -0,0 +1,9 @@
+import { box } from '../src/index';
+
+describe('Box', () => {
+	it('should have a __v property', () => {
+		const boxed = box(1);
+		expect('_value' in boxed).toBe(false);
+		expect(boxed.__v).toBe(1);
+	});
+});

e2e-test/webpack-custom/webpack.config.js

@@ -0,0 +1,24 @@
+module.exports = {
+	mode: 'development',
+	module: {
+		rules: [
+			{
+				test: /\.jsx?$/,
+				loader: 'babel-loader',
+				options: {
+					plugins: [
+						[
+							'babel-plugin-transform-rename-properties',
+							{ rename: { _value: '__v' } },
+						],
+					],
+				},
+			},
+		],
+	},
+	performance: {
+		hints: false,
+	},
+	devtool: 'inline-source-map',
+	stats: 'errors-only',
+};

e2e-test/webpack-default/package.json

@@ -0,0 +1,16 @@
+{
+	"name": "karmatic-e2e-webpack-default",
+	"description": "Test default webpack config in karmatic. Mildly complex src implementation to verify coverage works",
+	"private": true,
+	"scripts": {
+		"test": "cross-env NODE_PRESERVE_SYMLINKS_MAIN=1 NODE_PRESERVE_SYMLINKS=1 node ./node_modules/karmatic/dist/cli.js run",
+		"test:watch": "cross-env NODE_PRESERVE_SYMLINKS_MAIN=1 NODE_PRESERVE_SYMLINKS=1 node ./node_modules/karmatic/dist/cli.js --headless false"
+	},
+	"dependencies": {
+		"karmatic": "file:../..",
+		"webpack": "^4.44.1"
+	},
+	"devDependencies": {
+		"cross-env": "^7.0.2"
+	}
+}

fake-project/test/index.test.js → e2e-test/webpack-default/test/combine.test.js

@@ -1,25 +1,5 @@
 import { combine } from '../src/index';
 
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-
-describe('Basic test functions', () => {
-	it('should work', () => {
-		expect(1).toEqual(1);
-	});
-
-	it('should handle deep equality', () => {
-		expect({ foo: 1 }).toEqual({ foo: 1 });
-	});
-
-	it('should handle async tests', async () => {
-		let start = Date.now();
-		await sleep(100);
-
-		let now = Date.now();
-		expect(now - start).toBeGreaterThan(50);
-	});
-});
-
 describe('combine', () => {
 	it('should concatenate strings', () => {
 		expect(combine('a', 'b')).toBe('ab');

test/webpack/index.test.js → e2e-test/webpack-default/test/index.test.js

@@ -1,8 +1,6 @@
-import worker from 'workerize-loader!./fixture.worker.js';
-
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
-describe('demo', () => {
+describe('Basic test functions', () => {
 	it('should work', () => {
 		expect(1).toEqual(1);
 	});
@@ -18,10 +16,4 @@ describe('demo', () => {
 		let now = Date.now();
 		expect(now - start).toBeGreaterThan(50);
 	});
-
-	it('should do MAGIC', async () => {
-		let mod = worker();
-		expect(mod.foo).toEqual(jasmine.any(Function));
-		expect(await mod.foo()).toEqual(1);
-	});
 });

e2e-test/webpack-loader/index.js

@@ -0,0 +1,10 @@
+import createWorker from 'workerize-loader!./fixture.worker.js';
+
+let worker;
+export function getWorker() {
+	if (!worker) {
+		worker = createWorker();
+	}
+
+	return worker;
+}

e2e-test/webpack-loader/index.test.js

@@ -0,0 +1,10 @@
+import { getWorker } from './index';
+
+describe('demo', () => {
+	it('should do MAGIC', async () => {
+		let worker = getWorker();
+
+		expect(worker.foo).toEqual(jasmine.any(Function));
+		expect(await worker.foo()).toEqual(1);
+	});
+});

e2e-test/webpack-loader/package.json

@@ -0,0 +1,10 @@
+{
+	"name": "karmatic-e2e-webpack-workerize-loader",
+	"description": "Test customer webpack loader testing in karmatic",
+	"private": true,
+	"dependencies": {
+		"webpack": "^4.44.1",
+		"workerize-loader": "^1.3.0",
+		"karmatic": "file:../.."
+	}
+}

package.json

@@ -8,10 +8,11 @@
 	"scripts": {
 		"prepare": "npm t",
 		"build": "microbundle --target node -f cjs --no-compress src/index.js src/cli.js src/appender.js",
-		"test:build": "node ./dist/cli.js run",
-		"test:watch": "node ./dist/cli.js watch --headless false",
-		"prettier": "prettier --write './**/*.{js,json,yml,md}'",
-		"test": "prettier --check \"./**/*.{js,json,yml,md}\" && eslint src test && npm run -s build && npm run -s test:build",
+		"test:build": "cd e2e-test/webpack-default && npm test",
+		"test:watch": "cd e2e-test/webpack-default && npm run test:watch",
+		"test:e2e": "node ./scripts/run-e2e-tests.mjs",
+		"prettier": "prettier --write './**/*.{js,mjs,json,yml,md}'",
+		"test": "prettier --check \"./**/*.{js,mjs,json,yml,md}\" && eslint src e2e-test && npm run -s build && npm run -s test:e2e",
 		"release": "npm run -s prepare && npm test && git commit -am $npm_package_version && git tag $npm_package_version && git push && git push --tags && npm publish"
 	},
 	"eslintConfig": {
@@ -32,14 +33,14 @@
 	"repository": "developit/karmatic",
 	"license": "MIT",
 	"devDependencies": {
+		"@kristoferbaxter/async": "^1.0.0",
 		"eslint": "^7.3.0",
 		"eslint-config-developit": "^1.2.0",
 		"eslint-config-prettier": "^6.11.0",
 		"microbundle": "^0.12.2",
+		"micromatch": "^4.0.2",
 		"prettier": "^1.19.1",
-		"puppeteer": "^4.0.1",
-		"webpack": "^4.43.0",
-		"workerize-loader": "^1.3.0"
+		"puppeteer": "^4.0.1"
 	},
 	"dependencies": {
 		"@babel/core": "^7.11.0",