Merge pull request #12283 from ethereum/soltest-graceful-error-handling

Graceful error handling in soltest/isoltest + improved soltestAssert()

liblangutil/Exceptions.h

@@ -58,13 +58,13 @@ struct InvalidAstError: virtual util::Exception {};
 #endif
 
 #define solAssert_1(CONDITION) \
-	solAssert_2(CONDITION, "")
+	solAssert_2((CONDITION), "")
 
 #define solAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::langutil::InternalCompilerError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"Solidity assertion failed" \
 	)
 
@@ -77,13 +77,13 @@ struct InvalidAstError: virtual util::Exception {};
 #endif
 
 #define solUnimplementedAssert_1(CONDITION) \
-	solUnimplementedAssert_2(CONDITION, "")
+	solUnimplementedAssert_2((CONDITION), "")
 
 #define solUnimplementedAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::langutil::UnimplementedFeatureError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"Unimplemented feature" \
 	)
 
@@ -105,9 +105,9 @@ struct InvalidAstError: virtual util::Exception {};
 
 #define astAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::langutil::InvalidAstError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"AST assertion failed" \
 	)
 

libsmtutil/Exceptions.h

@@ -38,13 +38,13 @@ struct SMTLogicError: virtual util::Exception {};
 #endif
 
 #define smtAssert_1(CONDITION) \
-	smtAssert_2(CONDITION, "")
+	smtAssert_2((CONDITION), "")
 
 #define smtAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::smtutil::SMTLogicError, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"SMT assertion failed" \
 	)
 

libsolutil/Assertions.h

@@ -63,7 +63,7 @@ inline std::string stringOrDefault(std::string _string, std::string _defaultStri
 		if (!(_condition)) \
 			solThrow( \
 				_exceptionType, \
-				::solidity::util::assertions::stringOrDefault(_description, _defaultDescription) \
+				::solidity::util::assertions::stringOrDefault((_description), (_defaultDescription)) \
 			); \
 	} \
 	while (false)
@@ -72,6 +72,6 @@ inline std::string stringOrDefault(std::string _string, std::string _defaultStri
 /// Use it as assertThrow(1 == 1, ExceptionType, "Mathematics is wrong.");
 /// The second parameter must be an exception class (rather than an instance).
 #define assertThrow(_condition, _exceptionType, _description) \
-	assertThrowWithDefaultDescription(_condition, _exceptionType, _description, "Assertion failed")
+	assertThrowWithDefaultDescription((_condition), _exceptionType, (_description), "Assertion failed")
 
 }

libsolutil/Exceptions.h

@@ -48,7 +48,7 @@ struct Exception: virtual std::exception, virtual boost::exception
 #define solThrow(_exceptionType, _description) \
 	::boost::throw_exception( \
 		_exceptionType() << \
-		::solidity::util::errinfo_comment(_description) << \
+		::solidity::util::errinfo_comment((_description)) << \
 		::boost::throw_function(ETH_FUNC) << \
 		::boost::throw_file(__FILE__) << \
 		::boost::throw_line(__LINE__) \

libyul/Exceptions.h

@@ -63,13 +63,13 @@ struct StackTooDeepError: virtual YulException
 #endif
 
 #define yulAssert_1(CONDITION) \
-	yulAssert_2(CONDITION, "")
+	yulAssert_2((CONDITION), "")
 
 #define yulAssert_2(CONDITION, DESCRIPTION) \
 	assertThrowWithDefaultDescription( \
-		CONDITION, \
+		(CONDITION), \
 		::solidity::yul::YulAssertion, \
-		DESCRIPTION, \
+		(DESCRIPTION), \
 		"Yul assertion failed" \
 	)
 

test/Common.cpp

@@ -159,26 +159,33 @@ bool CommonOptions::parse(int argc, char const* const* argv)
 	po::variables_map arguments;
 	addOptions();
 
-	po::command_line_parser cmdLineParser(argc, argv);
-	cmdLineParser.options(options);
-	auto parsedOptions = cmdLineParser.run();
-	po::store(parsedOptions, arguments);
-	po::notify(arguments);
-
-	for (auto const& parsedOption: parsedOptions.options)
-		if (parsedOption.position_key >= 0)
-		{
-			if (
-				parsedOption.original_tokens.empty() ||
-				(parsedOption.original_tokens.size() == 1 && parsedOption.original_tokens.front().empty())
-			)
-				continue; // ignore empty options
-			std::stringstream errorMessage;
-			errorMessage << "Unrecognized option: ";
-			for (auto const& token: parsedOption.original_tokens)
-				errorMessage << token;
-			BOOST_THROW_EXCEPTION(std::runtime_error(errorMessage.str()));
-		}
+	try
+	{
+		po::command_line_parser cmdLineParser(argc, argv);
+		cmdLineParser.options(options);
+		auto parsedOptions = cmdLineParser.run();
+		po::store(parsedOptions, arguments);
+		po::notify(arguments);
+
+		for (auto const& parsedOption: parsedOptions.options)
+			if (parsedOption.position_key >= 0)
+			{
+				if (
+					parsedOption.original_tokens.empty() ||
+					(parsedOption.original_tokens.size() == 1 && parsedOption.original_tokens.front().empty())
+				)
+					continue; // ignore empty options
+				std::stringstream errorMessage;
+				errorMessage << "Unrecognized option: ";
+				for (auto const& token: parsedOption.original_tokens)
+					errorMessage << token;
+				BOOST_THROW_EXCEPTION(std::runtime_error(errorMessage.str()));
+			}
+	}
+	catch (po::error const& exception)
+	{
+		solThrow(ConfigException, exception.what());
+	}
 
 	if (vmPaths.empty())
 	{

test/Common.h

@@ -75,6 +75,9 @@ struct CommonOptions
 	langutil::EVMVersion evmVersion() const;
 
 	virtual void addOptions();
+	// @returns true if the program should continue, false if it should exit immediately without
+	// reporting an error.
+	// Throws ConfigException or std::runtime_error if parsing fails.
 	virtual bool parse(int argc, char const* const* argv);
 	// Throws a ConfigException on error
 	virtual void validate() const;

test/boostTest.cpp

@@ -204,88 +204,106 @@ int registerTests(
 	return numTestsAdded;
 }
 
-void initializeOptions()
+bool initializeOptions()
 {
 	auto const& suite = boost::unit_test::framework::master_test_suite();
 
 	auto options = std::make_unique<solidity::test::CommonOptions>();
-	solAssert(options->parse(suite.argc, suite.argv), "Failed to parse options!");
+	bool shouldContinue = options->parse(suite.argc, suite.argv);
+	if (!shouldContinue)
+		return false;
 	options->validate();
 
 	solidity::test::CommonOptions::setSingleton(std::move(options));
+	return true;
 }
 
 }
 
 // TODO: Prototype -- why isn't this declared in the boost headers?
 // TODO: replace this with a (global) fixture.
-test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] );
+test_suite* init_unit_test_suite(int /*argc*/, char* /*argv*/[]);
 
-test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
+test_suite* init_unit_test_suite(int /*argc*/, char* /*argv*/[])
 {
 	using namespace solidity::test;
 
 	master_test_suite_t& master = framework::master_test_suite();
 	master.p_name.value = "SolidityTests";
 
-	initializeOptions();
+	try
+	{
+		bool shouldContinue = initializeOptions();
+		if (!shouldContinue)
+			exit(EXIT_SUCCESS);
+
+		if (!solidity::test::loadVMs(solidity::test::CommonOptions::get()))
+			exit(EXIT_FAILURE);
 
-	if (!solidity::test::loadVMs(solidity::test::CommonOptions::get()))
-		exit(1);
+		if (solidity::test::CommonOptions::get().disableSemanticTests)
+			cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl;
 
-	if (solidity::test::CommonOptions::get().disableSemanticTests)
-		cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl;
+		if (!solidity::test::CommonOptions::get().enforceGasTest)
+			cout << endl << "WARNING :: Gas Cost Expectations are not being enforced" << endl << endl;
 
-	if (!solidity::test::CommonOptions::get().enforceGasTest)
-		cout << endl << "WARNING :: Gas Cost Expectations are not being enforced" << endl << endl;
+		Batcher batcher(CommonOptions::get().selectedBatch, CommonOptions::get().batches);
+		if (CommonOptions::get().batches > 1)
+			cout << "Batch " << CommonOptions::get().selectedBatch << " out of " << CommonOptions::get().batches << endl;
 
-	Batcher batcher(CommonOptions::get().selectedBatch, CommonOptions::get().batches);
-	if (CommonOptions::get().batches > 1)
-		cout << "Batch " << CommonOptions::get().selectedBatch << " out of " << CommonOptions::get().batches << endl;
+		// Batch the boost tests
+		BoostBatcher boostBatcher(batcher);
+		traverse_test_tree(master, boostBatcher, true);
 
-	// Batch the boost tests
-	BoostBatcher boostBatcher(batcher);
-	traverse_test_tree(master, boostBatcher, true);
+		// Include the interactive tests in the automatic tests as well
+		for (auto const& ts: g_interactiveTestsuites)
+		{
+			auto const& options = solidity::test::CommonOptions::get();
+
+			if (ts.smt && options.disableSMT)
+				continue;
+
+			if (ts.needsVM && solidity::test::CommonOptions::get().disableSemanticTests)
+				continue;
+
+			//TODO
+			//solAssert(
+			registerTests(
+				master,
+				options.testPath / ts.path,
+				ts.subpath,
+				options.enforceViaYul,
+				options.enforceCompileToEwasm,
+				ts.labels,
+				ts.testCaseCreator,
+				batcher
+			);
+			// > 0, std::string("no ") + ts.title + " tests found");
+		 }
 
-	// Include the interactive tests in the automatic tests as well
-	for (auto const& ts: g_interactiveTestsuites)
+		if (solidity::test::CommonOptions::get().disableSemanticTests)
+		{
+			for (auto suite: {
+				"ABIDecoderTest",
+				"ABIEncoderTest",
+				"SolidityAuctionRegistrar",
+				"SolidityWallet",
+				"GasMeterTests",
+				"GasCostTests",
+				"SolidityEndToEndTest",
+				"SolidityOptimizer"
+			})
+				removeTestSuite(suite);
+		}
+	}
+	catch (solidity::test::ConfigException const& exception)
 	{
-		auto const& options = solidity::test::CommonOptions::get();
-
-		if (ts.smt && options.disableSMT)
-			continue;
-
-		if (ts.needsVM && solidity::test::CommonOptions::get().disableSemanticTests)
-			continue;
-
-		//TODO
-		//solAssert(
-		registerTests(
-			master,
-			options.testPath / ts.path,
-			ts.subpath,
-			options.enforceViaYul,
-			options.enforceCompileToEwasm,
-			ts.labels,
-			ts.testCaseCreator,
-			batcher
-		);
-		// > 0, std::string("no ") + ts.title + " tests found");
+		cerr << exception.what() << endl;
+		exit(EXIT_FAILURE);
 	}
-
-	if (solidity::test::CommonOptions::get().disableSemanticTests)
+	catch (std::runtime_error const& exception)
 	{
-		for (auto suite: {
-			"ABIDecoderTest",
-			"ABIEncoderTest",
-			"SolidityAuctionRegistrar",
-			"SolidityWallet",
-			"GasMeterTests",
-			"GasCostTests",
-			"SolidityEndToEndTest",
-			"SolidityOptimizer"
-		})
-			removeTestSuite(suite);
+		cerr << exception.what() << endl;
+		exit(EXIT_FAILURE);
 	}
 
 	return nullptr;

test/libsolidity/util/SoltestErrors.h

@@ -15,20 +15,35 @@
 #pragma once
 
 #include <libsolutil/AnsiColorized.h>
+#include <libsolutil/Assertions.h>
 #include <libsolutil/CommonData.h>
 #include <libsolutil/Exceptions.h>
 
+#include <boost/preprocessor/cat.hpp>
+#include <boost/preprocessor/facilities/empty.hpp>
+#include <boost/preprocessor/facilities/overload.hpp>
+
 namespace solidity::frontend::test
 {
 
-#define soltestAssert(CONDITION, DESCRIPTION) \
-	do \
-	{ \
-		if (!(CONDITION)) \
-			BOOST_THROW_EXCEPTION(std::runtime_error(DESCRIPTION)); \
-	} \
-	while (false)
+struct InternalSoltestError: virtual util::Exception {};
+
+#if !BOOST_PP_VARIADICS_MSVC
+#define soltestAssert(...) BOOST_PP_OVERLOAD(soltestAssert_,__VA_ARGS__)(__VA_ARGS__)
+#else
+#define soltestAssert(...) BOOST_PP_CAT(BOOST_PP_OVERLOAD(soltestAssert_,__VA_ARGS__)(__VA_ARGS__),BOOST_PP_EMPTY())
+#endif
+
+#define soltestAssert_1(CONDITION) \
+	soltestAssert_2((CONDITION), "")
 
+#define soltestAssert_2(CONDITION, DESCRIPTION) \
+	assertThrowWithDefaultDescription( \
+		(CONDITION), \
+		::solidity::frontend::test::InternalSoltestError, \
+		(DESCRIPTION), \
+		"Soltest assertion failed" \
+	)
 
 class TestParserError: virtual public util::Exception
 {

test/tools/IsolTestOptions.cpp

@@ -75,17 +75,17 @@ void IsolTestOptions::addOptions()
 
 bool IsolTestOptions::parse(int _argc, char const* const* _argv)
 {
-	bool const res = CommonOptions::parse(_argc, _argv);
+	bool const shouldContinue = CommonOptions::parse(_argc, _argv);
 
-	if (showHelp || !res)
+	if (showHelp || !shouldContinue)
 	{
 		std::cout << options << std::endl;
 		return false;
 	}
 
 	enforceGasTest = enforceGasTest || (evmVersion() == langutil::EVMVersion{} && !useABIEncoderV1);
 
-	return res;
+	return shouldContinue;
 }
 
 void IsolTestOptions::validate() const