diff --git a/build-scripts/build.sh b/build-scripts/build.sh
index a713754c49efe..74807d81cd0d8 100755
--- a/build-scripts/build.sh
+++ b/build-scripts/build.sh
@@ -161,6 +161,18 @@ else
         fi
         wait -n
     fi
+
+    if [ -n "$TEST_STAGE" ]
+    then
+        # Run the tests one more time, without actually running any tests, just to verify that all
+        # the mod data can be successfully loaded
+
+        # Use a blacklist of mods that currently fail to load cleanly.  Hopefully this list will
+        # shrink over time.
+        blacklist=build-scripts/mod_test_blacklist
+        mods="$(./build-scripts/get_all_mods.py $blacklist)"
+        run_tests ./tests/cata_test --user-dir=all_modded --mods="$mods" '~*'
+    fi
 fi
 ccache --show-stats
 
diff --git a/build-scripts/get_all_mods.py b/build-scripts/get_all_mods.py
new file mode 100755
index 0000000000000..27f0e8cd55784
--- /dev/null
+++ b/build-scripts/get_all_mods.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+import sys
+import glob
+import json
+
+blacklist_filename, = sys.argv[1:]
+with open(blacklist_filename) as blacklist_file:
+    blacklist = {s.rstrip('\n') for s in blacklist_file.readlines()}
+
+mods = []
+
+for info in glob.glob('data/mods/*/modinfo.json'):
+    mod_info = json.load(open(info))
+    mods.extend(e["ident"] for e in mod_info if e["type"] == "MOD_INFO")
+
+mods_to_keep = [mod for mod in mods if mod not in blacklist]
+
+print(','.join(mods_to_keep))
diff --git a/build-scripts/mod_test_blacklist b/build-scripts/mod_test_blacklist
new file mode 100644
index 0000000000000..4286dca554c83
--- /dev/null
+++ b/build-scripts/mod_test_blacklist
@@ -0,0 +1,51 @@
+aftershock
+alt_map_key
+Animatronics
+Battery_Overhaul_Legacy_Mode
+blazemod
+cbm_slots
+classic_zombies
+crazy_cataclysm
+crt_expansion
+desertpack
+DinoMod
+ew_pack
+FIC_Weapons
+FujiStruct
+generic_guns
+Graphical_Overmap
+growable-pots
+Heavy miners
+hydroponics
+magiclysm
+manualbionicinstall
+mapgen_demo
+Medieval_Stuff
+MMA
+modular_turrets
+more_locations
+More_Survival_Tools
+mutant_npcs
+my_sweet_cataclysm
+national_guard_camp
+No_Anthills
+No_Bees
+no_faults
+no_filthy_clothing
+No_Fungi
+no_medieval_items
+no_npc_food
+No_Rail_Stations
+No_Triffids
+novitamins
+realguns
+safeautodoc
+Salvaged_Robots
+sees_player_hitbutton
+sees_player_retro
+sleepdeprivation
+speedydex
+stats_through_kills
+Tanks
+Tolerate_This
+Urban_Development
diff --git a/tests/test_main.cpp b/tests/test_main.cpp
index 9d71d809799cc..65d7d2db73153 100644
--- a/tests/test_main.cpp
+++ b/tests/test_main.cpp
@@ -41,6 +41,7 @@
 #include "worldfactory.h"
 #include "color.h"
 #include "options.h"
+#include "output.h"
 #include "pldata.h"
 #include "rng.h"
 #include "type_id.h"
@@ -68,26 +69,15 @@ static std::string extract_argument( std::vector<const char *> &arg_vec, const s
 
 static std::vector<mod_id> extract_mod_selection( std::vector<const char *> &arg_vec )
 {
-    std::vector<mod_id> ret;
     std::string mod_string = extract_argument( arg_vec, "--mods=" );
 
-    const char delim = ',';
-    size_t i = 0;
-    size_t pos = mod_string.find( delim );
-    if( pos == std::string::npos && !mod_string.empty() ) {
-        ret.emplace_back( mod_string );
-    }
-
-    while( pos != std::string::npos ) {
-        ret.emplace_back( mod_string.substr( i, pos - i ) );
-        i = ++pos;
-        pos = mod_string.find( delim, pos );
-
-        if( pos == std::string::npos ) {
-            ret.emplace_back( mod_string.substr( i, mod_string.length() ) );
+    std::vector<std::string> mod_names = string_split( mod_string, ',' );
+    std::vector<mod_id> ret;
+    for( const std::string mod_name : mod_names ) {
+        if( !mod_name.empty() ) {
+            ret.emplace_back( mod_name );
         }
     }
-
     return ret;
 }