diff --git a/build-scripts/build.sh b/build-scripts/build.sh
index a713754c49efe..74807d81cd0d8 100755
--- a/build-scripts/build.sh
+++ b/build-scripts/build.sh
@@ -161,6 +161,18 @@ else
         fi
         wait -n
     fi
+
+    if [ -n "$TEST_STAGE" ]
+    then
+        # Run the tests one more time, without actually running any tests, just to verify that all
+        # the mod data can be successfully loaded
+
+        # Use a blacklist of mods that currently fail to load cleanly.  Hopefully this list will
+        # shrink over time.
+        blacklist=build-scripts/mod_test_blacklist
+        mods="$(./build-scripts/get_all_mods.py $blacklist)"
+        run_tests ./tests/cata_test --user-dir=all_modded --mods="$mods" '~*'
+    fi
 fi
 ccache --show-stats
 
diff --git a/build-scripts/get_all_mods.py b/build-scripts/get_all_mods.py
new file mode 100755
index 0000000000000..27f0e8cd55784
--- /dev/null
+++ b/build-scripts/get_all_mods.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+import sys
+import glob
+import json
+
+blacklist_filename, = sys.argv[1:]
+with open(blacklist_filename) as blacklist_file:
+    blacklist = {s.rstrip('\n') for s in blacklist_file.readlines()}
+
+mods = []
+
+for info in glob.glob('data/mods/*/modinfo.json'):
+    mod_info = json.load(open(info))
+    mods.extend(e["ident"] for e in mod_info if e["type"] == "MOD_INFO")
+
+mods_to_keep = [mod for mod in mods if mod not in blacklist]
+
+print(','.join(mods_to_keep))
diff --git a/build-scripts/mod_test_blacklist b/build-scripts/mod_test_blacklist
new file mode 100644
index 0000000000000..4286dca554c83
--- /dev/null
+++ b/build-scripts/mod_test_blacklist
@@ -0,0 +1,51 @@
+aftershock
+alt_map_key
+Animatronics
+Battery_Overhaul_Legacy_Mode
+blazemod
+cbm_slots
+classic_zombies
+crazy_cataclysm
+crt_expansion
+desertpack
+DinoMod
+ew_pack
+FIC_Weapons
+FujiStruct
+generic_guns
+Graphical_Overmap
+growable-pots
+Heavy miners
+hydroponics
+magiclysm
+manualbionicinstall
+mapgen_demo
+Medieval_Stuff
+MMA
+modular_turrets
+more_locations
+More_Survival_Tools
+mutant_npcs
+my_sweet_cataclysm
+national_guard_camp
+No_Anthills
+No_Bees
+no_faults
+no_filthy_clothing
+No_Fungi
+no_medieval_items
+no_npc_food
+No_Rail_Stations
+No_Triffids
+novitamins
+realguns
+safeautodoc
+Salvaged_Robots
+sees_player_hitbutton
+sees_player_retro
+sleepdeprivation
+speedydex
+stats_through_kills
+Tanks
+Tolerate_This
+Urban_Development