diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index aa479d4..92b3ef3 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -12,9 +12,8 @@ jobs:
         with:
           node-version: 12
       - name: npm install, build, and test
+        # npm install runs build and test as well
         run: |
           npm install
-          npm run build --if-present
-          npm test
         env:
           CI: true
diff --git a/e2e-test/webpack-custom/package.json b/e2e-test/webpack-custom/package.json
new file mode 100644
index 0000000..b181fa6
--- /dev/null
+++ b/e2e-test/webpack-custom/package.json
@@ -0,0 +1,12 @@
+{
+	"name": "karmatic-e2e-webpack-custom",
+	"description": "Test custom webpack config in karmatic using a custom Babel config",
+	"private": true,
+	"dependencies": {
+		"@babel/core": "^7.10.3",
+		"babel-loader": "8.1.0",
+		"babel-plugin-transform-rename-properties": "^0.1.0",
+		"karmatic": "file:../..",
+		"webpack": "^4.44.1"
+	}
+}
diff --git a/e2e-test/webpack-custom/src/index.js b/e2e-test/webpack-custom/src/index.js
new file mode 100644
index 0000000..0dc0ae1
--- /dev/null
+++ b/e2e-test/webpack-custom/src/index.js
@@ -0,0 +1,3 @@
+export function box(value) {
+	return { _value: value };
+}
diff --git a/e2e-test/webpack-custom/test/index.test.js b/e2e-test/webpack-custom/test/index.test.js
new file mode 100644
index 0000000..e55386a
--- /dev/null
+++ b/e2e-test/webpack-custom/test/index.test.js
@@ -0,0 +1,9 @@
+import { box } from '../src/index';
+
+describe('Box', () => {
+	it('should have a __v property', () => {
+		const boxed = box(1);
+		expect('_value' in boxed).toBe(false);
+		expect(boxed.__v).toBe(1);
+	});
+});
diff --git a/e2e-test/webpack-custom/webpack.config.js b/e2e-test/webpack-custom/webpack.config.js
new file mode 100644
index 0000000..af1fa3a
--- /dev/null
+++ b/e2e-test/webpack-custom/webpack.config.js
@@ -0,0 +1,24 @@
+module.exports = {
+	mode: 'development',
+	module: {
+		rules: [
+			{
+				test: /\.jsx?$/,
+				loader: 'babel-loader',
+				options: {
+					plugins: [
+						[
+							'babel-plugin-transform-rename-properties',
+							{ rename: { _value: '__v' } },
+						],
+					],
+				},
+			},
+		],
+	},
+	performance: {
+		hints: false,
+	},
+	devtool: 'inline-source-map',
+	stats: 'errors-only',
+};
diff --git a/e2e-test/webpack-default/package.json b/e2e-test/webpack-default/package.json
new file mode 100644
index 0000000..beb5160
--- /dev/null
+++ b/e2e-test/webpack-default/package.json
@@ -0,0 +1,16 @@
+{
+	"name": "karmatic-e2e-webpack-default",
+	"description": "Test default webpack config in karmatic. Mildly complex src implementation to verify coverage works",
+	"private": true,
+	"scripts": {
+		"test": "cross-env NODE_PRESERVE_SYMLINKS_MAIN=1 NODE_PRESERVE_SYMLINKS=1 node ./node_modules/karmatic/dist/cli.js run",
+		"test:watch": "cross-env NODE_PRESERVE_SYMLINKS_MAIN=1 NODE_PRESERVE_SYMLINKS=1 node ./node_modules/karmatic/dist/cli.js --headless false"
+	},
+	"dependencies": {
+		"karmatic": "file:../..",
+		"webpack": "^4.44.1"
+	},
+	"devDependencies": {
+		"cross-env": "^7.0.2"
+	}
+}
diff --git a/fake-project/src/index.js b/e2e-test/webpack-default/src/index.js
similarity index 100%
rename from fake-project/src/index.js
rename to e2e-test/webpack-default/src/index.js
diff --git a/fake-project/test/index.test.js b/e2e-test/webpack-default/test/combine.test.js
similarity index 71%
rename from fake-project/test/index.test.js
rename to e2e-test/webpack-default/test/combine.test.js
index 9093c67..ab57c4c 100644
--- a/fake-project/test/index.test.js
+++ b/e2e-test/webpack-default/test/combine.test.js
@@ -1,25 +1,5 @@
 import { combine } from '../src/index';
 
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-
-describe('Basic test functions', () => {
-	it('should work', () => {
-		expect(1).toEqual(1);
-	});
-
-	it('should handle deep equality', () => {
-		expect({ foo: 1 }).toEqual({ foo: 1 });
-	});
-
-	it('should handle async tests', async () => {
-		let start = Date.now();
-		await sleep(100);
-
-		let now = Date.now();
-		expect(now - start).toBeGreaterThan(50);
-	});
-});
-
 describe('combine', () => {
 	it('should concatenate strings', () => {
 		expect(combine('a', 'b')).toBe('ab');
diff --git a/test/custom-pragma.test.js b/e2e-test/webpack-default/test/custom-pragma.test.js
similarity index 100%
rename from test/custom-pragma.test.js
rename to e2e-test/webpack-default/test/custom-pragma.test.js
diff --git a/test/webpack/index.test.js b/e2e-test/webpack-default/test/index.test.js
similarity index 61%
rename from test/webpack/index.test.js
rename to e2e-test/webpack-default/test/index.test.js
index 8b7c7be..59ddaa7 100644
--- a/test/webpack/index.test.js
+++ b/e2e-test/webpack-default/test/index.test.js
@@ -1,8 +1,6 @@
-import worker from 'workerize-loader!./fixture.worker.js';
-
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
-describe('demo', () => {
+describe('Basic test functions', () => {
 	it('should work', () => {
 		expect(1).toEqual(1);
 	});
@@ -18,10 +16,4 @@ describe('demo', () => {
 		let now = Date.now();
 		expect(now - start).toBeGreaterThan(50);
 	});
-
-	it('should do MAGIC', async () => {
-		let mod = worker();
-		expect(mod.foo).toEqual(jasmine.any(Function));
-		expect(await mod.foo()).toEqual(1);
-	});
 });
diff --git a/test/jest-style.test.js b/e2e-test/webpack-default/test/jest-style.test.js
similarity index 100%
rename from test/jest-style.test.js
rename to e2e-test/webpack-default/test/jest-style.test.js
diff --git a/test/webpack/fixture.worker.js b/e2e-test/webpack-loader/fixture.worker.js
similarity index 100%
rename from test/webpack/fixture.worker.js
rename to e2e-test/webpack-loader/fixture.worker.js
diff --git a/e2e-test/webpack-loader/index.js b/e2e-test/webpack-loader/index.js
new file mode 100644
index 0000000..20f622c
--- /dev/null
+++ b/e2e-test/webpack-loader/index.js
@@ -0,0 +1,10 @@
+import createWorker from 'workerize-loader!./fixture.worker.js';
+
+let worker;
+export function getWorker() {
+	if (!worker) {
+		worker = createWorker();
+	}
+
+	return worker;
+}
diff --git a/e2e-test/webpack-loader/index.test.js b/e2e-test/webpack-loader/index.test.js
new file mode 100644
index 0000000..dd416ae
--- /dev/null
+++ b/e2e-test/webpack-loader/index.test.js
@@ -0,0 +1,10 @@
+import { getWorker } from './index';
+
+describe('demo', () => {
+	it('should do MAGIC', async () => {
+		let worker = getWorker();
+
+		expect(worker.foo).toEqual(jasmine.any(Function));
+		expect(await worker.foo()).toEqual(1);
+	});
+});
diff --git a/e2e-test/webpack-loader/package.json b/e2e-test/webpack-loader/package.json
new file mode 100644
index 0000000..9e93bfa
--- /dev/null
+++ b/e2e-test/webpack-loader/package.json
@@ -0,0 +1,10 @@
+{
+	"name": "karmatic-e2e-webpack-workerize-loader",
+	"description": "Test customer webpack loader testing in karmatic",
+	"private": true,
+	"dependencies": {
+		"webpack": "^4.44.1",
+		"workerize-loader": "^1.3.0",
+		"karmatic": "file:../.."
+	}
+}
diff --git a/package.json b/package.json
index 481faf6..f1aafeb 100644
--- a/package.json
+++ b/package.json
@@ -8,10 +8,11 @@
 	"scripts": {
 		"prepare": "npm t",
 		"build": "microbundle --target node -f cjs --no-compress src/index.js src/cli.js src/appender.js",
-		"test:build": "node ./dist/cli.js run",
-		"test:watch": "node ./dist/cli.js watch --headless false",
-		"prettier": "prettier --write './**/*.{js,json,yml,md}'",
-		"test": "prettier --check \"./**/*.{js,json,yml,md}\" && eslint src test && npm run -s build && npm run -s test:build",
+		"test:build": "cd e2e-test/webpack-default && npm test",
+		"test:watch": "cd e2e-test/webpack-default && npm run test:watch",
+		"test:e2e": "node ./scripts/run-e2e-tests.mjs",
+		"prettier": "prettier --write './**/*.{js,mjs,json,yml,md}'",
+		"test": "prettier --check \"./**/*.{js,mjs,json,yml,md}\" && eslint src e2e-test && npm run -s build && npm run -s test:e2e",
 		"release": "npm run -s prepare && npm test && git commit -am $npm_package_version && git tag $npm_package_version && git push && git push --tags && npm publish"
 	},
 	"eslintConfig": {
@@ -32,14 +33,14 @@
 	"repository": "developit/karmatic",
 	"license": "MIT",
 	"devDependencies": {
+		"@kristoferbaxter/async": "^1.0.0",
 		"eslint": "^7.3.0",
 		"eslint-config-developit": "^1.2.0",
 		"eslint-config-prettier": "^6.11.0",
 		"microbundle": "^0.12.2",
+		"micromatch": "^4.0.2",
 		"prettier": "^1.19.1",
-		"puppeteer": "^4.0.1",
-		"webpack": "^4.43.0",
-		"workerize-loader": "^1.3.0"
+		"puppeteer": "^4.0.1"
 	},
 	"dependencies": {
 		"@babel/core": "^7.11.0",
diff --git a/scripts/run-e2e-tests.mjs b/scripts/run-e2e-tests.mjs
new file mode 100644
index 0000000..7bc6456
--- /dev/null
+++ b/scripts/run-e2e-tests.mjs
@@ -0,0 +1,297 @@
+import { execFile } from 'child_process';
+import { Transform } from 'stream';
+import { promises as fs } from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import chalk from 'chalk';
+import micromatch from 'micromatch';
+import { pool } from '@kristoferbaxter/async';
+
+// @ts-ignore
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = (...args) => path.join(__dirname, '..', ...args);
+const e2eRoot = (...args) => repoRoot('e2e-test', ...args);
+
+const isWindows = process.platform === 'win32';
+const npmCmd = isWindows ? 'npm.cmd' : 'npm';
+
+const info = chalk.blue;
+const error = chalk.red;
+
+async function fileExists(file) {
+	try {
+		return (await fs.stat(file)).isFile();
+	} catch (e) {}
+	return false;
+}
+
+/**
+ * Return a promise that resolves/rejects when the child process exits
+ * @param {import('child_process').ChildProcess} childProcess
+ * @param {(code: number, signal: string) => boolean} [isSuccess]
+ */
+async function onExit(childProcess, isSuccess) {
+	if (!isSuccess) {
+		isSuccess = (code, signal) => code === 0 || signal == 'SIGINT';
+	}
+
+	return new Promise((resolve, reject) => {
+		childProcess.once('exit', (code, signal) => {
+			if (isSuccess(code, signal)) {
+				resolve();
+			} else {
+				reject(new Error('Child process exited with error code: ' + code));
+			}
+		});
+
+		childProcess.once('error', (err) => {
+			reject(err);
+		});
+	});
+}
+
+const noisyLog = /No repository field|No license field|SKIPPING OPTIONAL DEPENDENCY|You must install peer dependencies yourself/;
+
+/**
+ * Prefix every line in a stream with the given prefix
+ * @param {string} prefix
+ */
+function createPrefixTransform(prefix) {
+	let incompleteLine = '';
+	return new Transform({
+		transform(chunk, encoding, callback) {
+			try {
+				// @ts-ignore
+				chunk = encoding == 'buffer' ? chunk.toString() : chunk;
+				// console.log('CHUNK:', JSON.stringify(chunk));
+
+				const lines = chunk.split('\n');
+
+				// Prepend any incomplete parts from the previous chunk
+				lines[0] = incompleteLine + lines[0];
+				incompleteLine = '';
+
+				for (let i = 0; i < lines.length; i++) {
+					let line = lines[i];
+					if (i == lines.length - 1) {
+						// If chunk contains complete lines (i.e. ends in with a newline
+						// character), then the last line in the lines array will be an
+						// empty string. If chunk contains an incomplete line (i.e. does not
+						// end in a newline character), then the last line will the
+						// beginning of next line
+						incompleteLine = line;
+					} else if (line && line.match(noisyLog) == null) {
+						line = `${prefix} ${line}`;
+						this.push(line + '\n');
+					}
+				}
+
+				callback();
+			} catch (error) {
+				return callback(error);
+			}
+		},
+		flush(callback) {
+			// console.log('FLUSH:', JSON.stringify(incompleteLine));
+			if (incompleteLine) {
+				let chunk = incompleteLine;
+				incompleteLine = '';
+				callback(null, chunk);
+			} else {
+				callback(null);
+			}
+		},
+	});
+}
+
+function getOpts(cwd) {
+	// # Why Enable NODE_PRESERVE_SYMLINKS and NODE_PRESERVE_SYMLINKS_MAIN
+	//
+	// When running karmatic in the e2e-test folder, when need to ensure that
+	// Node's `require` follows the symlinks we have set up so that karmatic sees
+	// the e2e-test local install of webpack.
+	//
+	// Karmatic's behavior changes based on what the user has installed. So, to
+	// validate these different kinds of installations our tests runs karmatic
+	// with different peer deps installed.
+	//
+	// To simulate this, each e2e-test package has a local install of karmatic.
+	// This local install actually just points to the current repository using a
+	// file reference: `file:../..`. This reference instructs npm to install the
+	// referenced folder as a symbolic link in the e2e-test's node_modules folder:
+	//
+	//    e2e-test/webpack-default/node_modules/karmatic/dist/cli.js
+	//
+	// However, by default, NodeJS's `require` function resolves packages from a
+	// module's real location, not symbolic location. So when running karmatic
+	// from the symbolic directory above, NodeJS would see that file's location as
+	// the following:
+	//
+	//    dist/cli.js
+	//
+	// This behavior causes us problems because we want Node's require function to
+	// use the symbolic location so it can find whatever peer deps the e2e test
+	// has installed (e.g. webpack). Those peer deps aren't available at the root
+	// of the repo, so resolving from `dist/cli.js` would never see webpack
+	// installed at `e2e-test/webpack-default/node_modules/webpack`.
+	//
+	// To fix this, we enable the `NODE_PRESERVE_SYMLINKS` and
+	// `NODE_PRESERVE_SYMLINKS_MAIN` environment variables instructing Node's
+	// require function to use a module's symbolic path to resolve modules. This
+	// behavior means that when karmatic tries to require `webpack` in the e2e
+	// tests, it will attempt to find `webpack` from
+	//
+	//    e2e-test/webpack-default/node_modules/karmatic/dist/cli.js
+	//
+	// and should locate the `webpack` installation at
+	//
+	//    e2e-test/webpack-default/node_modules/webpack
+	//
+	// Documentation:
+	// https://nodejs.org/dist/latest/docs/api/cli.html#cli_node_preserve_symlinks_1
+
+	return {
+		cwd,
+		env: {
+			...process.env,
+			NODE_PRESERVE_SYMLINKS: '1',
+			NODE_PRESERVE_SYMLINKS_MAIN: '1',
+		},
+	};
+}
+
+/** Run `npm install` in the given directory */
+async function npmInstall(cwd, prefix) {
+	const name = path.basename(cwd);
+	console.log(`${info(prefix)} Installing packages for "${name}"...`);
+
+	const cp = execFile(npmCmd, ['install', '--no-fund'], { cwd });
+
+	prefix = prefix || `[${name}]`;
+	cp.stdout.pipe(createPrefixTransform(info(prefix))).pipe(process.stdout);
+	cp.stderr.pipe(createPrefixTransform(error(prefix))).pipe(process.stderr);
+
+	await onExit(cp);
+}
+
+/**
+ * @param {string} projectPath
+ * @param {string} prefix
+ * @returns {Promise<() => Promise<void>>}
+ */
+async function setupTests(projectPath, prefix) {
+	const name = path.basename(projectPath);
+	const log = (...msgs) => console.log(`${info(prefix)}`, ...msgs);
+
+	log(`Beginning E2E test at`, projectPath);
+	const pkgJsonPath = path.join(projectPath, 'package.json');
+	if (!(await fileExists(pkgJsonPath))) {
+		prefix = error(prefix);
+		console.error(
+			`${prefix} Could not locate package.json for "${name}". Ensure every e2e test has a package.json defined.`
+		);
+		console.error(`${prefix} Expected to find one at "${pkgJsonPath}".`);
+		throw new Error(`Could not locate package.json for "${name}".`);
+	}
+
+	const pkg = JSON.parse(await fs.readFile(pkgJsonPath, 'utf8'));
+
+	if (pkg.dependencies.karmatic == null) {
+		log(`Updating package.json with karmatic path...`);
+		const relativePath = path.relative(projectPath, repoRoot());
+		pkg.dependencies.karmatic = `file:` + relativePath.replace(/\\/g, '/');
+		const newContents = JSON.stringify(pkg, null, 2);
+		await fs.writeFile(pkgJsonPath, newContents, 'utf8');
+	}
+
+	await npmInstall(projectPath, prefix);
+
+	return async () => {
+		let cmd, args, opts;
+		if (pkg.scripts && pkg.scripts.test) {
+			cmd = npmCmd;
+			args = ['test'];
+			opts = { cwd: projectPath };
+			log(`Running npm test...`);
+		} else {
+			cmd = process.execPath;
+			args = ['node_modules/karmatic/dist/cli.js', 'run'];
+			opts = getOpts(projectPath);
+			log(`Running karmatic...`);
+		}
+
+		const cp = execFile(cmd, args, opts);
+		cp.stdout.pipe(createPrefixTransform(info(prefix))).pipe(process.stdout);
+		cp.stderr.pipe(createPrefixTransform(error(prefix))).pipe(process.stderr);
+
+		try {
+			await onExit(cp);
+		} catch (e) {
+			process.exitCode = 1;
+			console.error(error(prefix) + ` Test run failed: ${e.message}`);
+		}
+
+		// TODO: validate coverage/lcov.info is not empty
+	};
+}
+
+/**
+ * @param {string[]} args
+ */
+async function main(args) {
+	if (args.includes('--help')) {
+		console.log(
+			`\nRun Karmatic E2E Tests.\n\n` +
+				`Accepts globs of matching e2e tests (directory names of the e2e-test folder) as arguments.\n` +
+				`Example: node ./scripts/run-e2e-tests.mjs default-*\n`
+		);
+
+		return;
+	}
+
+	process.on('exit', (code) => {
+		if (code !== 0) {
+			console.log(
+				error('A fatal error occurred. Check the logs above for details.')
+			);
+		}
+	});
+
+	let matchers = args.map((glob) => micromatch.matcher(glob));
+	let entries = await fs.readdir(e2eRoot(), { withFileTypes: true });
+	let projects = entries
+		.filter((p) => p.isDirectory)
+		.map((p) => p.name)
+		.filter((name) =>
+			matchers.length !== 0 ? matchers.some((isMatch) => isMatch(name)) : true
+		);
+
+	const length = projects.reduce((max, name) => Math.max(max, name.length), 0);
+	const getPrefix = (name) => `[${name.padEnd(length)}]`;
+
+	console.log(
+		args.length === 0
+			? `Setting up all E2E tests.`
+			: `Setting up selected E2E tests: ${projects.join(', ')}`
+	);
+
+	try {
+		// Run npm installs serially to avoid any weird behavior since we are
+		// installing using symlinks
+		let runners = [];
+		for (let project of projects) {
+			runners.push(await setupTests(e2eRoot(project), getPrefix(project)));
+		}
+
+		console.log('Running karmatic...');
+		// await pool(runners, (run) => run());
+		for (let run of runners) {
+			await run();
+		}
+	} catch (e) {
+		process.exitCode = 1;
+		console.error(e);
+	}
+}
+
+main(process.argv.slice(2));
diff --git a/src/cli.js b/src/cli.js
index 32ca63b..806939b 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -64,6 +64,6 @@ function run(str, opts, isWatch) {
 						'\n'
 				);
 			}
-			process.exit(err.code || 1);
+			process.exit(typeof err.code == 'number' ? err.code : 1);
 		});
 }
diff --git a/src/webpack.js b/src/webpack.js
index a6db999..dc887fb 100644
--- a/src/webpack.js
+++ b/src/webpack.js
@@ -116,6 +116,10 @@ export function addWebpackConfig(karmaConfig, pkg, options) {
 		mode: webpackConfig.mode || 'development',
 		module: {
 			// @TODO check webpack version and use loaders VS rules as the key here appropriately:
+			//
+			// TODO: Consider adding coverage as a separate babel-loader so that
+			// regardless if the user provides their own babel plugins, coverage still
+			// works
 			rules: loaders
 				.concat(
 					!getLoader((rule) =>