diff --git a/test/generate.js b/test/generate.js
index 3db6ea0d4cdf..384bd95b841e 100644
--- a/test/generate.js
+++ b/test/generate.js
@@ -49,6 +49,10 @@ describe( 'generate', () => {
 
 		const config = loadConfig( dir );
 
+		if ( config.solo && process.env.CI ) {
+			throw new Error( 'Forgot to remove `solo: true` from test' );
+		}
+
 		( config.skip ? it.skip : config.solo ? it.only : it )( dir, () => {
 			let compiled;
 
diff --git a/test/generator/component-yield-multiple-in-if/_config.js b/test/generator/component-yield-multiple-in-if/_config.js
index 7ba8c6c497f9..58e0ecfabc6a 100644
--- a/test/generator/component-yield-multiple-in-if/_config.js
+++ b/test/generator/component-yield-multiple-in-if/_config.js
@@ -1,6 +1,4 @@
 export default {
-	solo: true,
-
 	html: `
 		<div><p class='widget'>Hello</p></div>
 	`,
diff --git a/test/parse.js b/test/parse.js
index dc4d8f2e6adb..166a17609b3c 100644
--- a/test/parse.js
+++ b/test/parse.js
@@ -8,6 +8,10 @@ describe( 'parse', () => {
 
 		const solo = exists( `test/parser/${dir}/solo` );
 
+		if ( solo && process.env.CI ) {
+			throw new Error( 'Forgot to remove `solo: true` from test' );
+		}
+
 		( solo ? it.only : it )( dir, () => {
 			const input = fs.readFileSync( `test/parser/${dir}/input.html`, 'utf-8' ).replace( /\s+$/, '' );
 
diff --git a/test/sourcemaps.js b/test/sourcemaps.js
index 3417681629c8..bafaf11a8ab6 100644
--- a/test/sourcemaps.js
+++ b/test/sourcemaps.js
@@ -10,6 +10,10 @@ describe( 'sourcemaps', () => {
 
 		const solo = exists( `test/sourcemaps/${dir}/solo` );
 
+		if ( solo && process.env.CI ) {
+			throw new Error( 'Forgot to remove `solo: true` from test' );
+		}
+
 		( solo ? it.only : it )( dir, () => {
 			const input = fs.readFileSync( `test/sourcemaps/${dir}/input.html`, 'utf-8' ).replace( /\s+$/, '' );
 			const { code, map } = svelte.compile( input );
diff --git a/test/ssr.js b/test/ssr.js
index 828b10c797c2..ebac203de3ca 100644
--- a/test/ssr.js
+++ b/test/ssr.js
@@ -26,6 +26,10 @@ describe( 'ssr', () => {
 
 		const solo = exists( `test/server-side-rendering/${dir}/solo` );
 
+		if ( solo && process.env.CI ) {
+			throw new Error( 'Forgot to remove `solo: true` from test' );
+		}
+
 		( solo ? it.only : it )( dir, () => {
 			const component = require( `./server-side-rendering/${dir}/main.html` );
 
diff --git a/test/validate.js b/test/validate.js
index 6708074338e6..1545d51701bf 100644
--- a/test/validate.js
+++ b/test/validate.js
@@ -8,6 +8,10 @@ describe( 'validate', () => {
 
 		const solo = exists( `test/validator/${dir}/solo` );
 
+		if ( solo && process.env.CI ) {
+			throw new Error( 'Forgot to remove `solo: true` from test' );
+		}
+
 		( solo ? it.only : it )( dir, () => {
 			const filename = `test/validator/${dir}/input.html`;
 			const input = fs.readFileSync( filename, 'utf-8' ).replace( /\s+$/, '' );