From 30a6e2dbf88bac8f1b1b5634b3b0f8aa9549aaff Mon Sep 17 00:00:00 2001 From: Dan Albert Date: Wed, 28 Sep 2016 17:01:34 -0700 Subject: [PATCH] Add support for is_negative_test. Note that this is different from build_broken. Use build_broken to indicate a bug and use is_negative_test to indicate a test that should fail if things are working. Also note that check_broken and is_negative_test can be layered. If a build is expected to fail, but doesn't for armeabi, the test_config could contain: def is_negative_test(): return True def build_broken(abi, api, toolchain): if abi == 'armeabi': return abi, bug_url return None, None Test: Added a test that uses this in the next patch. Bug: None Change-Id: Ib12e974db9bc58d9e28ec97646cfb81c208cc4cd --- tests/testlib.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/tests/testlib.py b/tests/testlib.py index e6726b90..ca0763bb 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -266,6 +266,15 @@ def _fixup_expected_failure(result, config, bug): return result +def _fixup_negative_test(result): + if isinstance(result, Failure): + return Success(result.test_name) + elif isinstance(result, Success): + return Failure(result.test_name, 'negative test case succeeded') + else: # Skipped, UnexpectedSuccess, or ExpectedFailure. + return result + + def _run_test(suite, test, out_dir, test_filters): """Runs a given test according to the given filters. @@ -286,8 +295,10 @@ def _run_test(suite, test, out_dir, test_filters): message = 'test unsupported for {}'.format(config) return suite, Skipped(test.name, message), [] - config, bug = test.check_broken() result, additional_tests = test.run(out_dir, test_filters) + if test.is_negative_test(): + result = _fixup_negative_test(result) + config, bug = test.check_broken() if config is not None: # We need to check change each pass/fail to either an # ExpectedFailure or an UnexpectedSuccess as necessary. @@ -455,6 +466,9 @@ def check_broken(self): def check_unsupported(self): return None + def is_negative_test(self): + return False + def run(self, out_dir, test_filters): # We need a subdirectory named for our test to handle the case where # multiple awk tests share names for test cases. If run simultaneously, @@ -561,6 +575,29 @@ def extra_cmake_flags(): def extra_ndk_build_flags(): """Returns extra flags that should be passed to ndk-build.""" return [] + + @staticmethod + def is_negative_test(): + """Returns True if this test should pass if the build fails. + + Note that this is different from build_broken. Use build_broken to + indicate a bug and use is_negative_test to indicate a test that + should fail if things are working. + + Also note that check_broken and is_negative_test can be layered. If + a build is expected to fail, but doesn't for armeabi, the + test_config could contain: + + def is_negative_test(): + return True + + + def build_broken(abi, api, toolchain): + if abi == 'armeabi': + return abi, bug_url + return None, None + """ + return False # pylint: enable=unused-argument def __init__(self, file_path): @@ -595,6 +632,11 @@ def __init__(self, file_path): ntc = self.NullTestConfig self.extra_ndk_build_flags = ntc.extra_ndk_build_flags + try: + self.is_negative_test = self.module.is_negative_test + except AttributeError: + self.is_negative_test = self.NullTestConfig.is_negative_test + @classmethod def from_test_dir(cls, test_dir): path = os.path.join(test_dir, 'test_config.py') @@ -635,6 +677,18 @@ def __init__(self, file_path): except AttributeError: self.run_unsupported = self.NullTestConfig.run_unsupported + try: + _ = self.module.is_negative_test + # If the build is expected to fail, then it should just be a build + # test since the test should never be run. + # + # If the run is expected to fail, just fix the test to pass for + # thatr case. Gtest death tests can handle the more complicated + # cases. + raise RuntimeError('is_negative_test is invalid for device tests') + except AttributeError: + pass + def _run_build_sh_test(test_name, build_dir, test_dir, ndk_build_flags, abi, platform, toolchain): @@ -757,6 +811,9 @@ def check_unsupported(self): return self.get_test_config().build_unsupported( self.abi, self.platform, self.toolchain) + def is_negative_test(self): + return self.get_test_config().is_negative_test() + def get_extra_cmake_flags(self): return self.get_test_config().extra_cmake_flags() @@ -948,6 +1005,9 @@ def check_unsupported(self): return self.get_test_config().build_unsupported( self.abi, self.platform, self.toolchain) + def is_negative_test(self): + return False + def run(self, out_dir, test_filters): raise NotImplementedError @@ -1101,6 +1161,9 @@ def check_unsupported(self): return self.get_test_config().run_unsupported( self.abi, self.device_api, self.toolchain, self.case_name) + def is_negative_test(self): + return False + def run(self, out_dir, test_filters): cmd = 'cd {} && LD_LIBRARY_PATH={} ./{} 2>&1'.format( self.device_dir, self.device_dir, self.case_name)