From bd46902570f376ddac19a123e09d83942106d998 Mon Sep 17 00:00:00 2001 From: MMirbach Date: Mon, 4 Jan 2021 10:10:08 +0200 Subject: [PATCH 1/7] print issues --- unit-tests/run-tests.py | 383 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 unit-tests/run-tests.py diff --git a/unit-tests/run-tests.py b/unit-tests/run-tests.py new file mode 100644 index 0000000000..9fe091037d --- /dev/null +++ b/unit-tests/run-tests.py @@ -0,0 +1,383 @@ +#!python3 + +# License: Apache 2.0. See LICENSE file in root directory. +# Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +import sys, os, subprocess, locale, re, platform, getopt +from abc import ABC + +def usage(): + ourname = os.path.basename(sys.argv[0]) + print( 'Syntax: ' + ourname + ' ' ) + print( ' Both parameters are optional') + print( ' dir: the directory holding the executable test files to run. If not specified we try to assume') + print( ' one ourselves from build directory. This fails if there are 2 versions of tests, example:') + print( ' tests built in Debug and test built in Release') + print( ' regex: run all tests that fit . If not specified, run all tests') + print( 'Options:' ) + print( ' --debug Turn on debugging information' ) + print( ' -v, --verbose Errors will dump the log to stdout' ) + print( ' -q, --quiet Suppress output; rely on exit status (0=no failures)' ) + sys.exit(2) +def debug(*args): + pass + +def stream_has_color( stream ): + if not hasattr(stream, "isatty"): + return False + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + curses.setupterm() + return curses.tigetnum( "colors" ) > 2 + except: + # guess false in case of error + return False + +# Set up the default output system; if not a terminal, disable colors! +if stream_has_color( sys.stdout ): + red = '\033[91m' + gray = '\033[90m' + reset = '\033[0m' + cr = '\033[G' + clear_eos = '\033[J' + clear_eol = '\033[K' + _progress = '' + def out(*args): + print( *args, end = clear_eol + '\n' ) + global _progress + if len(_progress): + progress( *_progress ) + def progress(*args): + print( *args, end = clear_eol + '\r' ) + global _progress + _progress = args +else: + red = gray = reset = cr = clear_eos = '' + def out(*args): + print( *args ) + def progress(*args): + print( *args ) + +n_errors = 0 +def error(*args): + global red + global reset + out( red + '-E-' + reset, *args ) + global n_errors + n_errors = n_errors + 1 +def info(*args): + out( '-I-', *args) + +def filesin( root ): + # Yield all files found in root, using relative names ('root/a' would be yielded as 'a') + for (path,subdirs,leafs) in os.walk( root ): + for leaf in leafs: + # We have to stick to Unix conventions because CMake on Windows is fubar... + yield os.path.relpath( path + '/' + leaf, root ).replace( '\\', '/' ) + +def find( dir, mask ): + pattern = re.compile( mask ) + for leaf in filesin( dir ): + if pattern.search( leaf ): + debug(leaf) + yield leaf + +# get os and directories for future use +# NOTE: WSL will read as 'Linux' but the build is Windows-based! +system = platform.system() +if system == 'Linux' and "microsoft" not in platform.uname()[3].lower(): + linux = True +else: + linux = False + +current_dir = os.path.dirname(os.path.abspath(__file__)) +# this script is located in librealsense/unit-tests, so one directory up is the main repository +librealsense = os.path.dirname(current_dir) + +# function for checking if a file is an executable + +def is_executable(path_to_test): + if linux: + return os.access(path_to_test, os.X_OK) + else: + return path_to_test.endswith('.exe') + +# Parse command-line: +try: + opts,args = getopt.getopt( sys.argv[1:], 'hvq', + longopts = [ 'help', 'verbose', 'debug', 'quiet' ]) +except getopt.GetoptError as err: + error( err ) # something like "option -a not recognized" + usage() +verbose = False +for opt,arg in opts: + if opt in ('-h','--help'): + usage() + elif opt in ('--debug'): + def debug(*args): + global gray + global reset + out( gray + '-D-', *args, reset ) + elif opt in ('-v','--verbose'): + verbose = True + elif opt in ('-q','--quiet'): + def out(*args): + pass +if len(args) > 2: + usage() +regex = None +target = None +if len(args) == 2: + if not os.path.isdir(args[0]): + usage() + target = args[0] + regex = args[1] +if len(args) == 1: + if os.path.isdir( args[0] ): + target = args[0] + else: + regex = args[0] +# Trying to assume target directory from inside build directory. Only works if there is only one location with tests +if not target: + build = librealsense + os.sep + 'build' + for executable in find(build, '(^|/)test-.*'): + if not is_executable(executable): + continue + dir_with_test = build + os.sep + os.path.dirname(executable) + if target and target != dir_with_test: + error("Two versions of tests found!") + usage() + target = dir_with_test + +if target: + logdir = target + '/unit-tests' +else: # no test executables were found. We put the logs directly in build directory + logdir = librealsense + os.sep + 'build' +os.makedirs( logdir, exist_ok = True ) +n_tests = 0 + +def run( cmd, stdout = None ): + debug( 'Running:', cmd ) + handle = None + try: + if stdout and stdout != subprocess.PIPE: + handle = open( stdout, "w" ) + stdout = handle + rv = subprocess.run( cmd, + stdout = stdout, + stderr = subprocess.STDOUT, + universal_newlines = True, + check = True) + result = rv.stdout + if not result: + result = [] + else: + result = result.split( '\n' ) + return result + finally: + if handle: + handle.close() + +# Python scripts should be able to find the pyrealsense2 .pyd or else they won't work. We don't know +# if the user (Travis included) has pyrealsense2 installed but even if so, we want to use the one we compiled. +# we search the librealsense repository for the .pyd file (.so file in linux) +pyrs = "" +if linux: + for so in find(librealsense, '(^|/)pyrealsense2.*\.so$'): + pyrs = so +else: + for pyd in find(librealsense, '(^|/)pyrealsense2.*\.pyd$'): + pyrs = pyd + +if pyrs: + # After use of find, pyrs contains the path from librealsense to the pyrealsense that was found + # We append it to the librealsense path to get an absolute path to the file to add to PYTHONPATH so it can be found by the tests + pyrs_path = librealsense + os.sep + pyrs + # We need to add the directory not the file itself + pyrs_path = os.path.dirname(pyrs_path) + # Add the necessary path to the PYTHONPATH environment variable so python will look for modules there + os.environ["PYTHONPATH"] = pyrs_path + # We also need to add the path to the python packages that the tests use + os.environ["PYTHONPATH"] += os.pathsep + (current_dir + os.sep + "py") + # We can simply change `sys.path` but any child python scripts won't see it. We change the environment instead. + +def remove_newlines (lines): + for line in lines: + if line[-1] == '\n': + line = line[:-1] # excluding the endline + yield line + +def grep_( pattern, lines, context ): + index = 0 + matches = 0 + for line in lines: + index = index + 1 + match = pattern.search( line ) + if match: + context['index'] = index + context['line'] = line + context['match'] = match + yield context + matches = matches + 1 + if matches: + del context['index'] + del context['line'] + del context['match'] + +def grep( expr, *args ): + #debug( f'grep {expr} {args}' ) + pattern = re.compile( expr ) + context = dict() + for filename in args: + context['filename'] = filename + with open( filename, errors = 'ignore' ) as file: + for line in grep_( pattern, remove_newlines( file ), context ): + yield context + +def cat( filename ): + with open( filename, errors = 'ignore' ) as file: + for line in remove_newlines( file ): + out( line ) + +def check_log_for_fails(log, testname, exe): + # Normal logs are expected to have in last line: + # "All tests passed (11 assertions in 1 test case)" + # Tests that have failures, however, will show: + # "test cases: 1 | 1 failed + # assertions: 9 | 6 passed | 3 failed" + global verbose + for ctx in grep( r'^test cases:\s*(\d+) \|\s*(\d+) (passed|failed)', log ): + m = ctx['match'] + total = int(m.group(1)) + passed = int(m.group(2)) + if m.group(3) == 'failed': + # "test cases: 1 | 1 failed" + passed = total - passed + if passed < total: + if total == 1 or passed == 0: + desc = 'failed' + else: + desc = str(total - passed) + ' of ' + str(total) + ' failed' + + if verbose: + error( red + testname + reset + ': ' + desc ) + info( 'Executable:', exe ) + info( 'Log: >>>' ) + out() + cat( log ) + out( '<<<' ) + else: + error( red + testname + reset + ': ' + desc + '; see ' + log ) + return True + return False + +# definition of classes for tests +class Test(ABC): + """ + Abstract class for a test. Holds the name of the test, the log file for it and the command line needed to run it + """ + def __init__(self, testname, command, executable): + self.testname = testname + self.command = command + self.executable = executable + self.log = logdir + '/' + testname + '.log' + + def run_test(self): + global n_tests + progress(self.testname, '>', self.log, '...') + n_tests += 1 + try: + run( self.command, stdout=self.log ) + except FileNotFoundError: + error(red + self.testname + reset + ': executable not found! (' + self.executable + ')') + except subprocess.CalledProcessError as cpe: + if not check_log_for_fails(self.log, self.testname, self.executable): + # An unexpected error occurred + error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')') + +class PyTest(Test): + """ + Class for python tests + """ + def __init__(self, testname, path_to_test): + """ + :param testname: name of the test + :param path_to_test: the relative path from the current directory to the path + """ + global current_dir + + if linux: + cmd = ["python3", path_to_test] + else: + cmd = ["py", "-3", path_to_test] + + Test.__init__(self, testname, cmd, current_dir + path_to_test) + +class CTest(Test): + """ + Class for c/cpp tests + """ + def __init__(self, testname, exe): + """ + :param testname: name of the test + :param exe: full path to executable + """ + Test.__init__(self, testname, exe, exe) + +def get_tests(): + global regex, target, pyrs, current_dir + if regex: + pattern = re.compile(regex) + if target: + # In Linux, the build targets are located elsewhere than on Windows + # Go over all the tests from a "manifest" we take from the result of the last CMake + # run (rather than, for example, looking for test-* in the build-directory): + if linux: + manifestfile = target + '/CMakeFiles/TargetDirectories.txt' + else: + manifestfile = target + '/../CMakeFiles/TargetDirectories.txt' + for manifest_ctx in grep(r'(?<=unit-tests/build/)\S+(?=/CMakeFiles/test-\S+.dir$)', manifestfile): + # We need to first create the test name so we can see if it fits the regex + testdir = manifest_ctx['match'].group(0) # "log/internal/test-all" + testparent = os.path.dirname(testdir) # "log/internal" + if testparent: + testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(testdir)[5:] # "test-log-internal-all" + else: + testname = testdir # no parent folder so we get "test-all" + + if regex and not pattern.search( testname ): + continue + + if linux: + exe = target + '/unit-tests/build/' + testdir + '/' + testname + else: + exe = target + '/' + testname + '.exe' + + yield CTest(testname, exe) + + # If we run python tests with no .pyd/.so file they will crash. Therefore we only run them if such a file was found + if pyrs: + # unit-test scripts are in the same directory as this script + for py_test in find(current_dir, '(^|/)test-.*\.py'): + testparent = os.path.dirname(py_test) # "log/internal" <- "log/internal/test-all.py" + if testparent: + testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(py_test)[5:-3] # remove .py + else: + testname = os.path.basename(py_test)[:-3] + + if regex and not pattern.search( testname ): + continue + + yield PyTest(testname, py_test) + +for test in get_tests(): + test.run_test() + +progress() +if n_errors: + out( red + str(n_errors) + reset + ' of ' + str(n_tests) + ' test(s) failed!' + clear_eos ) + sys.exit(1) +out( str(n_tests) + ' unit-test(s) completed successfully' + clear_eos ) +sys.exit(0) \ No newline at end of file From 0af15f552980924fae5987c81dc74a40b9dc6157 Mon Sep 17 00:00:00 2001 From: MMirbach Date: Mon, 4 Jan 2021 16:32:17 +0200 Subject: [PATCH 2/7] run-unit-test update --- unit-tests/run-tests.py | 383 ----------------------------------- unit-tests/run-unit-tests.py | 294 +++++++++++++++------------ 2 files changed, 165 insertions(+), 512 deletions(-) delete mode 100644 unit-tests/run-tests.py diff --git a/unit-tests/run-tests.py b/unit-tests/run-tests.py deleted file mode 100644 index 9fe091037d..0000000000 --- a/unit-tests/run-tests.py +++ /dev/null @@ -1,383 +0,0 @@ -#!python3 - -# License: Apache 2.0. See LICENSE file in root directory. -# Copyright(c) 2020 Intel Corporation. All Rights Reserved. - -import sys, os, subprocess, locale, re, platform, getopt -from abc import ABC - -def usage(): - ourname = os.path.basename(sys.argv[0]) - print( 'Syntax: ' + ourname + ' ' ) - print( ' Both parameters are optional') - print( ' dir: the directory holding the executable test files to run. If not specified we try to assume') - print( ' one ourselves from build directory. This fails if there are 2 versions of tests, example:') - print( ' tests built in Debug and test built in Release') - print( ' regex: run all tests that fit . If not specified, run all tests') - print( 'Options:' ) - print( ' --debug Turn on debugging information' ) - print( ' -v, --verbose Errors will dump the log to stdout' ) - print( ' -q, --quiet Suppress output; rely on exit status (0=no failures)' ) - sys.exit(2) -def debug(*args): - pass - -def stream_has_color( stream ): - if not hasattr(stream, "isatty"): - return False - if not stream.isatty(): - return False # auto color only on TTYs - try: - import curses - curses.setupterm() - return curses.tigetnum( "colors" ) > 2 - except: - # guess false in case of error - return False - -# Set up the default output system; if not a terminal, disable colors! -if stream_has_color( sys.stdout ): - red = '\033[91m' - gray = '\033[90m' - reset = '\033[0m' - cr = '\033[G' - clear_eos = '\033[J' - clear_eol = '\033[K' - _progress = '' - def out(*args): - print( *args, end = clear_eol + '\n' ) - global _progress - if len(_progress): - progress( *_progress ) - def progress(*args): - print( *args, end = clear_eol + '\r' ) - global _progress - _progress = args -else: - red = gray = reset = cr = clear_eos = '' - def out(*args): - print( *args ) - def progress(*args): - print( *args ) - -n_errors = 0 -def error(*args): - global red - global reset - out( red + '-E-' + reset, *args ) - global n_errors - n_errors = n_errors + 1 -def info(*args): - out( '-I-', *args) - -def filesin( root ): - # Yield all files found in root, using relative names ('root/a' would be yielded as 'a') - for (path,subdirs,leafs) in os.walk( root ): - for leaf in leafs: - # We have to stick to Unix conventions because CMake on Windows is fubar... - yield os.path.relpath( path + '/' + leaf, root ).replace( '\\', '/' ) - -def find( dir, mask ): - pattern = re.compile( mask ) - for leaf in filesin( dir ): - if pattern.search( leaf ): - debug(leaf) - yield leaf - -# get os and directories for future use -# NOTE: WSL will read as 'Linux' but the build is Windows-based! -system = platform.system() -if system == 'Linux' and "microsoft" not in platform.uname()[3].lower(): - linux = True -else: - linux = False - -current_dir = os.path.dirname(os.path.abspath(__file__)) -# this script is located in librealsense/unit-tests, so one directory up is the main repository -librealsense = os.path.dirname(current_dir) - -# function for checking if a file is an executable - -def is_executable(path_to_test): - if linux: - return os.access(path_to_test, os.X_OK) - else: - return path_to_test.endswith('.exe') - -# Parse command-line: -try: - opts,args = getopt.getopt( sys.argv[1:], 'hvq', - longopts = [ 'help', 'verbose', 'debug', 'quiet' ]) -except getopt.GetoptError as err: - error( err ) # something like "option -a not recognized" - usage() -verbose = False -for opt,arg in opts: - if opt in ('-h','--help'): - usage() - elif opt in ('--debug'): - def debug(*args): - global gray - global reset - out( gray + '-D-', *args, reset ) - elif opt in ('-v','--verbose'): - verbose = True - elif opt in ('-q','--quiet'): - def out(*args): - pass -if len(args) > 2: - usage() -regex = None -target = None -if len(args) == 2: - if not os.path.isdir(args[0]): - usage() - target = args[0] - regex = args[1] -if len(args) == 1: - if os.path.isdir( args[0] ): - target = args[0] - else: - regex = args[0] -# Trying to assume target directory from inside build directory. Only works if there is only one location with tests -if not target: - build = librealsense + os.sep + 'build' - for executable in find(build, '(^|/)test-.*'): - if not is_executable(executable): - continue - dir_with_test = build + os.sep + os.path.dirname(executable) - if target and target != dir_with_test: - error("Two versions of tests found!") - usage() - target = dir_with_test - -if target: - logdir = target + '/unit-tests' -else: # no test executables were found. We put the logs directly in build directory - logdir = librealsense + os.sep + 'build' -os.makedirs( logdir, exist_ok = True ) -n_tests = 0 - -def run( cmd, stdout = None ): - debug( 'Running:', cmd ) - handle = None - try: - if stdout and stdout != subprocess.PIPE: - handle = open( stdout, "w" ) - stdout = handle - rv = subprocess.run( cmd, - stdout = stdout, - stderr = subprocess.STDOUT, - universal_newlines = True, - check = True) - result = rv.stdout - if not result: - result = [] - else: - result = result.split( '\n' ) - return result - finally: - if handle: - handle.close() - -# Python scripts should be able to find the pyrealsense2 .pyd or else they won't work. We don't know -# if the user (Travis included) has pyrealsense2 installed but even if so, we want to use the one we compiled. -# we search the librealsense repository for the .pyd file (.so file in linux) -pyrs = "" -if linux: - for so in find(librealsense, '(^|/)pyrealsense2.*\.so$'): - pyrs = so -else: - for pyd in find(librealsense, '(^|/)pyrealsense2.*\.pyd$'): - pyrs = pyd - -if pyrs: - # After use of find, pyrs contains the path from librealsense to the pyrealsense that was found - # We append it to the librealsense path to get an absolute path to the file to add to PYTHONPATH so it can be found by the tests - pyrs_path = librealsense + os.sep + pyrs - # We need to add the directory not the file itself - pyrs_path = os.path.dirname(pyrs_path) - # Add the necessary path to the PYTHONPATH environment variable so python will look for modules there - os.environ["PYTHONPATH"] = pyrs_path - # We also need to add the path to the python packages that the tests use - os.environ["PYTHONPATH"] += os.pathsep + (current_dir + os.sep + "py") - # We can simply change `sys.path` but any child python scripts won't see it. We change the environment instead. - -def remove_newlines (lines): - for line in lines: - if line[-1] == '\n': - line = line[:-1] # excluding the endline - yield line - -def grep_( pattern, lines, context ): - index = 0 - matches = 0 - for line in lines: - index = index + 1 - match = pattern.search( line ) - if match: - context['index'] = index - context['line'] = line - context['match'] = match - yield context - matches = matches + 1 - if matches: - del context['index'] - del context['line'] - del context['match'] - -def grep( expr, *args ): - #debug( f'grep {expr} {args}' ) - pattern = re.compile( expr ) - context = dict() - for filename in args: - context['filename'] = filename - with open( filename, errors = 'ignore' ) as file: - for line in grep_( pattern, remove_newlines( file ), context ): - yield context - -def cat( filename ): - with open( filename, errors = 'ignore' ) as file: - for line in remove_newlines( file ): - out( line ) - -def check_log_for_fails(log, testname, exe): - # Normal logs are expected to have in last line: - # "All tests passed (11 assertions in 1 test case)" - # Tests that have failures, however, will show: - # "test cases: 1 | 1 failed - # assertions: 9 | 6 passed | 3 failed" - global verbose - for ctx in grep( r'^test cases:\s*(\d+) \|\s*(\d+) (passed|failed)', log ): - m = ctx['match'] - total = int(m.group(1)) - passed = int(m.group(2)) - if m.group(3) == 'failed': - # "test cases: 1 | 1 failed" - passed = total - passed - if passed < total: - if total == 1 or passed == 0: - desc = 'failed' - else: - desc = str(total - passed) + ' of ' + str(total) + ' failed' - - if verbose: - error( red + testname + reset + ': ' + desc ) - info( 'Executable:', exe ) - info( 'Log: >>>' ) - out() - cat( log ) - out( '<<<' ) - else: - error( red + testname + reset + ': ' + desc + '; see ' + log ) - return True - return False - -# definition of classes for tests -class Test(ABC): - """ - Abstract class for a test. Holds the name of the test, the log file for it and the command line needed to run it - """ - def __init__(self, testname, command, executable): - self.testname = testname - self.command = command - self.executable = executable - self.log = logdir + '/' + testname + '.log' - - def run_test(self): - global n_tests - progress(self.testname, '>', self.log, '...') - n_tests += 1 - try: - run( self.command, stdout=self.log ) - except FileNotFoundError: - error(red + self.testname + reset + ': executable not found! (' + self.executable + ')') - except subprocess.CalledProcessError as cpe: - if not check_log_for_fails(self.log, self.testname, self.executable): - # An unexpected error occurred - error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')') - -class PyTest(Test): - """ - Class for python tests - """ - def __init__(self, testname, path_to_test): - """ - :param testname: name of the test - :param path_to_test: the relative path from the current directory to the path - """ - global current_dir - - if linux: - cmd = ["python3", path_to_test] - else: - cmd = ["py", "-3", path_to_test] - - Test.__init__(self, testname, cmd, current_dir + path_to_test) - -class CTest(Test): - """ - Class for c/cpp tests - """ - def __init__(self, testname, exe): - """ - :param testname: name of the test - :param exe: full path to executable - """ - Test.__init__(self, testname, exe, exe) - -def get_tests(): - global regex, target, pyrs, current_dir - if regex: - pattern = re.compile(regex) - if target: - # In Linux, the build targets are located elsewhere than on Windows - # Go over all the tests from a "manifest" we take from the result of the last CMake - # run (rather than, for example, looking for test-* in the build-directory): - if linux: - manifestfile = target + '/CMakeFiles/TargetDirectories.txt' - else: - manifestfile = target + '/../CMakeFiles/TargetDirectories.txt' - for manifest_ctx in grep(r'(?<=unit-tests/build/)\S+(?=/CMakeFiles/test-\S+.dir$)', manifestfile): - # We need to first create the test name so we can see if it fits the regex - testdir = manifest_ctx['match'].group(0) # "log/internal/test-all" - testparent = os.path.dirname(testdir) # "log/internal" - if testparent: - testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(testdir)[5:] # "test-log-internal-all" - else: - testname = testdir # no parent folder so we get "test-all" - - if regex and not pattern.search( testname ): - continue - - if linux: - exe = target + '/unit-tests/build/' + testdir + '/' + testname - else: - exe = target + '/' + testname + '.exe' - - yield CTest(testname, exe) - - # If we run python tests with no .pyd/.so file they will crash. Therefore we only run them if such a file was found - if pyrs: - # unit-test scripts are in the same directory as this script - for py_test in find(current_dir, '(^|/)test-.*\.py'): - testparent = os.path.dirname(py_test) # "log/internal" <- "log/internal/test-all.py" - if testparent: - testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(py_test)[5:-3] # remove .py - else: - testname = os.path.basename(py_test)[:-3] - - if regex and not pattern.search( testname ): - continue - - yield PyTest(testname, py_test) - -for test in get_tests(): - test.run_test() - -progress() -if n_errors: - out( red + str(n_errors) + reset + ' of ' + str(n_tests) + ' test(s) failed!' + clear_eos ) - sys.exit(1) -out( str(n_tests) + ' unit-test(s) completed successfully' + clear_eos ) -sys.exit(0) \ No newline at end of file diff --git a/unit-tests/run-unit-tests.py b/unit-tests/run-unit-tests.py index 209956cfd9..7f57915872 100644 --- a/unit-tests/run-unit-tests.py +++ b/unit-tests/run-unit-tests.py @@ -4,22 +4,21 @@ # Copyright(c) 2020 Intel Corporation. All Rights Reserved. import sys, os, subprocess, locale, re, platform, getopt +from abc import ABC def usage(): ourname = os.path.basename(sys.argv[0]) - print( 'Syntax: ' + ourname + ' ' ) - print( ' If given a directory, run all unit-tests in $dir (in Windows: .../build/Release; in Linux: .../build' ) - print( ' Test logs are kept in /unit-tests/.log' ) - print( ' If given a regular expression, run all python tests in unit-tests directory that fit , to stdout' ) + print( 'Syntax: ' + ourname + ' [options] [dir]' ) + print( ' dir: the directory holding the executable tests to run (default to the build directory') print( 'Options:' ) print( ' --debug Turn on debugging information' ) print( ' -v, --verbose Errors will dump the log to stdout' ) print( ' -q, --quiet Suppress output; rely on exit status (0=no failures)' ) + print( ' -r, --regex run all tests that fit the immediately following regular expression') sys.exit(2) def debug(*args): pass - def stream_has_color( stream ): if not hasattr(stream, "isatty"): return False @@ -56,7 +55,7 @@ def progress(*args): def out(*args): print( *args ) def progress(*args): - print( *args ) + print( *args, end='\r' ) n_errors = 0 def error(*args): @@ -68,15 +67,49 @@ def error(*args): def info(*args): out( '-I-', *args) +def filesin( root ): + # Yield all files found in root, using relative names ('root/a' would be yielded as 'a') + for (path,subdirs,leafs) in os.walk( root ): + for leaf in leafs: + # We have to stick to Unix conventions because CMake on Windows is fubar... + yield os.path.relpath( path + '/' + leaf, root ).replace( '\\', '/' ) + +def find( dir, mask ): + pattern = re.compile( mask ) + for leaf in filesin( dir ): + if pattern.search( leaf ): + debug(leaf) + yield leaf + +# get os and directories for future use +# NOTE: WSL will read as 'Linux' but the build is Windows-based! +system = platform.system() +if system == 'Linux' and "microsoft" not in platform.uname()[3].lower(): + linux = True +else: + linux = False + +current_dir = os.path.dirname(os.path.abspath(__file__)) +# this script is located in librealsense/unit-tests, so one directory up is the main repository +librealsense = os.path.dirname(current_dir) + +# function for checking if a file is an executable + +def is_executable(path_to_test): + if linux: + return os.access(path_to_test, os.X_OK) + else: + return path_to_test.endswith('.exe') # Parse command-line: try: - opts,args = getopt.getopt( sys.argv[1:], 'hvq', - longopts = [ 'help', 'verbose', 'debug', 'quiet' ]) + opts,args = getopt.getopt( sys.argv[1:], 'hvqr', + longopts = [ 'help', 'verbose', 'debug', 'quiet', 'regex' ]) except getopt.GetoptError as err: error( err ) # something like "option -a not recognized" usage() verbose = False +regex = None for opt,arg in opts: if opt in ('-h','--help'): usage() @@ -90,8 +123,35 @@ def debug(*args): elif opt in ('-q','--quiet'): def out(*args): pass -if len(args) != 1: + elif opt in ('-r', '--regex'): + regex = args[0] + del args[0] +if len(args) > 1: usage() +target = None +if len(args) == 1: + if os.path.isdir( args[0] ): + target = args[0] + else: + usage() +# Trying to assume target directory from inside build directory. Only works if there is only one location with tests +if not target: + build = librealsense + os.sep + 'build' + for executable in find(build, '(^|/)test-.*'): + if not is_executable(executable): + continue + dir_with_test = build + os.sep + os.path.dirname(executable) + if target and target != dir_with_test: + error("Found executable tests in 2 directories:", target, "and", dir_with_test, ". Can't default to directory") + usage() + target = dir_with_test + +if target: + logdir = target + '/unit-tests' +else: # no test executables were found. We put the logs directly in build directory + logdir = librealsense + os.sep + 'build' +os.makedirs( logdir, exist_ok = True ) +n_tests = 0 def run( cmd, stdout = None ): debug( 'Running:', cmd ) @@ -115,31 +175,6 @@ def run( cmd, stdout = None ): if handle: handle.close() -def filesin( root ): - # Yield all files found in root, using relative names ('root/a' would be yielded as 'a') - for (path,subdirs,leafs) in os.walk( root ): - for leaf in leafs: - # We have to stick to Unix conventions because CMake on Windows is fubar... - yield os.path.relpath( path + '/' + leaf, root ).replace( '\\', '/' ) - -def find( dir, mask ): - pattern = re.compile( mask ) - for leaf in filesin( dir ): - if pattern.search( leaf ): - debug(leaf) - yield leaf - -# NOTE: WSL will read as 'Linux' but the build is Windows-based! -system = platform.system() -if system == 'Linux' and "microsoft" not in platform.uname()[3].lower(): - linux = True -else: - linux = False - -current_dir = os.path.dirname(os.path.abspath(__file__)) -# this script is located in librealsense/unit-tests, so one directory up is the main repository -librealsense = os.path.dirname(current_dir) - # Python scripts should be able to find the pyrealsense2 .pyd or else they won't work. We don't know # if the user (Travis included) has pyrealsense2 installed but even if so, we want to use the one we compiled. # we search the librealsense repository for the .pyd file (.so file in linux) @@ -163,37 +198,6 @@ def find( dir, mask ): os.environ["PYTHONPATH"] += os.pathsep + (current_dir + os.sep + "py") # We can simply change `sys.path` but any child python scripts won't see it. We change the environment instead. -target = args[0] - -# If a regular expression (not a directory) is passed in, find the test(s) and run -# them directly -if not os.path.isdir( target ): - if not pyrs: - error( "Python wrappers (pyrealsense2*." + pyrs + ") not found" ) - usage() - n_tests = 0 - for py_test in find(current_dir, target): - n_tests += 1 - progress( py_test + ' ...' ) - if linux: - cmd = ["python3"] - else: - cmd = ["py", "-3"] - if sys.flags.verbose: - cmd += ["-v"] - cmd += [current_dir + os.sep + py_test] - try: - run( cmd ) - except subprocess.CalledProcessError as cpe: - error( cpe ) - error( red + py_test + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')' ) - if n_errors: - sys.exit(1) - if not n_tests: - error( "No tests found matching: " + target ) - usage() - sys.exit(0) - def remove_newlines (lines): for line in lines: if line[-1] == '\n': @@ -264,79 +268,111 @@ def check_log_for_fails(log, testname, exe): return True return False -logdir = target + '/unit-tests' -os.makedirs( logdir, exist_ok = True ) -n_tests = 0 - -# In Linux, the build targets are located elsewhere than on Windows -# Go over all the tests from a "manifest" we take from the result of the last CMake -# run (rather than, for example, looking for test-* in the build-directory): -if linux: - manifestfile = target + '/CMakeFiles/TargetDirectories.txt' -else: - manifestfile = target + '/../CMakeFiles/TargetDirectories.txt' - -for manifest_ctx in grep( r'(?<=unit-tests/build/)\S+(?=/CMakeFiles/test-\S+.dir$)', manifestfile ): - - testdir = manifest_ctx['match'].group(0) # "log/internal/test-all" - testparent = os.path.dirname(testdir) # "log/internal" - if testparent: - testname = 'test-' + testparent.replace( '/', '-' ) + '-' + os.path.basename(testdir)[5:] # "test-log-internal-all" - else: - testname = testdir # no parent folder so we get "test-all" - if linux: - exe = target + '/unit-tests/build/' + testdir + '/' + testname - else: - exe = target + '/' + testname + '.exe' - log = logdir + '/' + testname + '.log' +# definition of classes for tests +class Test(ABC): + """ + Abstract class for a test. Holds the name of the test, the log file for it and the command line needed to run it + """ + def __init__(self, testname, command, executable): + self.testname = testname + self.command = command + self.executable = executable + self.log = logdir + '/' + testname + '.log' - progress( testname, '>', log, '...' ) - n_tests += 1 - try: - run( [exe], stdout=log ) - except FileNotFoundError: - error( red + testname + reset + ': executable not found! (' + exe + ')' ) - continue - except subprocess.CalledProcessError as cpe: - if not check_log_for_fails(log, testname, exe): - # An unexpected error occurred - error( red + testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')' ) - -# If we run python tests with no .pyd/.so file they will crash. Therefore we only run them if such a file was found -if pyrs: - # unit-test scripts are in the same directory as this script - for py_test in find(current_dir, '(^|/)test-.*\.py'): - - testdir = py_test[:-3] # "log/internal/test-all" <- "log/internal/test-all.py" - testparent = os.path.dirname(testdir) # same as for cpp files - if testparent: - testname = 'test-' + testparent.replace( '/', '-' ) + '-' + os.path.basename(testdir)[5:] - else: - testname = testdir - - log = logdir + '/' + testname + '.log' - - progress( testname, '>', log, '...' ) + def run_test(self): + global n_tests + progress(self.testname, '>', self.log, '...') n_tests += 1 - test_path = current_dir + os.sep + py_test - if linux: - cmd = ["python3", test_path] - else: - cmd = ["py","-3", test_path] try: - run( cmd, stdout=log ) + run( self.command, stdout=self.log ) except FileNotFoundError: - error( red + testname + reset + ': file not found! (' + test_path + ')' ) - continue + error(red + self.testname + reset + ': executable not found! (' + self.executable + ')') except subprocess.CalledProcessError as cpe: - if not check_log_for_fails(log, testname, py_test): + if not check_log_for_fails(self.log, self.testname, self.executable): # An unexpected error occurred - cat(log) - error(red + testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')') + error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')') + +class PyTest(Test): + """ + Class for python tests + """ + def __init__(self, testname, path_to_test): + """ + :param testname: name of the test + :param path_to_test: the relative path from the current directory to the path + """ + global current_dir + + if linux: + cmd = ["python3", path_to_test] + else: + cmd = ["py", "-3", path_to_test] + + Test.__init__(self, testname, cmd, current_dir + path_to_test) + +class CTest(Test): + """ + Class for c/cpp tests + """ + def __init__(self, testname, exe): + """ + :param testname: name of the test + :param exe: full path to executable + """ + Test.__init__(self, testname, exe, exe) + +def get_tests(): + global regex, target, pyrs, current_dir + if regex: + pattern = re.compile(regex) + if target: + # In Linux, the build targets are located elsewhere than on Windows + # Go over all the tests from a "manifest" we take from the result of the last CMake + # run (rather than, for example, looking for test-* in the build-directory): + if linux: + manifestfile = target + '/CMakeFiles/TargetDirectories.txt' + else: + manifestfile = target + '/../CMakeFiles/TargetDirectories.txt' + for manifest_ctx in grep(r'(?<=unit-tests/build/)\S+(?=/CMakeFiles/test-\S+.dir$)', manifestfile): + # We need to first create the test name so we can see if it fits the regex + testdir = manifest_ctx['match'].group(0) # "log/internal/test-all" + testparent = os.path.dirname(testdir) # "log/internal" + if testparent: + testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(testdir)[5:] # "test-log-internal-all" + else: + testname = testdir # no parent folder so we get "test-all" + + if regex and not pattern.search( testname ): + continue + + if linux: + exe = target + '/unit-tests/build/' + testdir + '/' + testname + else: + exe = target + '/' + testname + '.exe' + + yield CTest(testname, exe) + + # If we run python tests with no .pyd/.so file they will crash. Therefore we only run them if such a file was found + if pyrs: + # unit-test scripts are in the same directory as this script + for py_test in find(current_dir, '(^|/)test-.*\.py'): + testparent = os.path.dirname(py_test) # "log/internal" <- "log/internal/test-all.py" + if testparent: + testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(py_test)[5:-3] # remove .py + else: + testname = os.path.basename(py_test)[:-3] + + if regex and not pattern.search( testname ): + continue + + yield PyTest(testname, py_test) + +for test in get_tests(): + test.run_test() progress() if n_errors: out( red + str(n_errors) + reset + ' of ' + str(n_tests) + ' test(s) failed!' + clear_eos ) sys.exit(1) out( str(n_tests) + ' unit-test(s) completed successfully' + clear_eos ) -sys.exit(0) +sys.exit(0) \ No newline at end of file From 825171605c04f95aae2364e193336f43f4c1740f Mon Sep 17 00:00:00 2001 From: MMirbach Date: Tue, 5 Jan 2021 14:09:08 +0200 Subject: [PATCH 3/7] implemented comments from PR --- unit-tests/run-unit-tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/unit-tests/run-unit-tests.py b/unit-tests/run-unit-tests.py index 7f57915872..a6dc2e89f9 100644 --- a/unit-tests/run-unit-tests.py +++ b/unit-tests/run-unit-tests.py @@ -14,7 +14,7 @@ def usage(): print( ' --debug Turn on debugging information' ) print( ' -v, --verbose Errors will dump the log to stdout' ) print( ' -q, --quiet Suppress output; rely on exit status (0=no failures)' ) - print( ' -r, --regex run all tests that fit the immediately following regular expression') + print( ' -r, --regex run all tests that fit the following regular expression') sys.exit(2) def debug(*args): pass @@ -123,9 +123,8 @@ def debug(*args): elif opt in ('-q','--quiet'): def out(*args): pass - elif opt in ('-r', '--regex'): - regex = args[0] - del args[0] + elif opt[0] in ('-r', '--regex'): + regex = opt[1] if len(args) > 1: usage() target = None From 234f482f61637e2317b8824a7a08cb890cba1b99 Mon Sep 17 00:00:00 2001 From: MMirbach Date: Tue, 5 Jan 2021 17:04:49 +0200 Subject: [PATCH 4/7] comments and name changes --- unit-tests/run-unit-tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/unit-tests/run-unit-tests.py b/unit-tests/run-unit-tests.py index a6dc2e89f9..0ab0ee0f70 100644 --- a/unit-tests/run-unit-tests.py +++ b/unit-tests/run-unit-tests.py @@ -146,7 +146,7 @@ def out(*args): target = dir_with_test if target: - logdir = target + '/unit-tests' + logdir = target + os.sep + 'unit-tests' else: # no test executables were found. We put the logs directly in build directory logdir = librealsense + os.sep + 'build' os.makedirs( logdir, exist_ok = True ) @@ -268,7 +268,7 @@ def check_log_for_fails(log, testname, exe): return False # definition of classes for tests -class Test(ABC): +class Test(ABC): # Abstract Base Class """ Abstract class for a test. Holds the name of the test, the log file for it and the command line needed to run it """ @@ -300,7 +300,7 @@ def __init__(self, testname, path_to_test): :param testname: name of the test :param path_to_test: the relative path from the current directory to the path """ - global current_dir + global current_dir, linux if linux: cmd = ["python3", path_to_test] @@ -309,9 +309,9 @@ def __init__(self, testname, path_to_test): Test.__init__(self, testname, cmd, current_dir + path_to_test) -class CTest(Test): +class ExeTest(Test): """ - Class for c/cpp tests + Class for c/cpp tests """ def __init__(self, testname, exe): """ @@ -321,7 +321,7 @@ def __init__(self, testname, exe): Test.__init__(self, testname, exe, exe) def get_tests(): - global regex, target, pyrs, current_dir + global regex, target, pyrs, current_dir, linux if regex: pattern = re.compile(regex) if target: @@ -349,7 +349,7 @@ def get_tests(): else: exe = target + '/' + testname + '.exe' - yield CTest(testname, exe) + yield ExeTest(testname, exe) # If we run python tests with no .pyd/.so file they will crash. Therefore we only run them if such a file was found if pyrs: From 7d32fd4ff89fa963b9e0bbb164b7a2674eabf578 Mon Sep 17 00:00:00 2001 From: MMirbach Date: Mon, 11 Jan 2021 13:08:13 +0200 Subject: [PATCH 5/7] changed Test classes structure --- unit-tests/run-unit-tests.py | 69 ++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/unit-tests/run-unit-tests.py b/unit-tests/run-unit-tests.py index 0ab0ee0f70..4bb3e151b6 100644 --- a/unit-tests/run-unit-tests.py +++ b/unit-tests/run-unit-tests.py @@ -4,7 +4,7 @@ # Copyright(c) 2020 Intel Corporation. All Rights Reserved. import sys, os, subprocess, locale, re, platform, getopt -from abc import ABC +from abc import ABC, abstractmethod def usage(): ourname = os.path.basename(sys.argv[0]) @@ -55,7 +55,7 @@ def progress(*args): def out(*args): print( *args ) def progress(*args): - print( *args, end='\r' ) + print( *args ) n_errors = 0 def error(*args): @@ -103,7 +103,7 @@ def is_executable(path_to_test): # Parse command-line: try: - opts,args = getopt.getopt( sys.argv[1:], 'hvqr', + opts,args = getopt.getopt( sys.argv[1:], 'hvqr:', longopts = [ 'help', 'verbose', 'debug', 'quiet', 'regex' ]) except getopt.GetoptError as err: error( err ) # something like "option -a not recognized" @@ -152,7 +152,8 @@ def out(*args): os.makedirs( logdir, exist_ok = True ) n_tests = 0 -def run( cmd, stdout = None ): +# wrapper function for subprocess.run +def subprocess_run(cmd, stdout = None): debug( 'Running:', cmd ) handle = None try: @@ -272,24 +273,12 @@ class Test(ABC): # Abstract Base Class """ Abstract class for a test. Holds the name of the test, the log file for it and the command line needed to run it """ - def __init__(self, testname, command, executable): + def __init__(self, testname): self.testname = testname - self.command = command - self.executable = executable - self.log = logdir + '/' + testname + '.log' + @abstractmethod def run_test(self): - global n_tests - progress(self.testname, '>', self.log, '...') - n_tests += 1 - try: - run( self.command, stdout=self.log ) - except FileNotFoundError: - error(red + self.testname + reset + ': executable not found! (' + self.executable + ')') - except subprocess.CalledProcessError as cpe: - if not check_log_for_fails(self.log, self.testname, self.executable): - # An unexpected error occurred - error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')') + pass class PyTest(Test): """ @@ -300,14 +289,25 @@ def __init__(self, testname, path_to_test): :param testname: name of the test :param path_to_test: the relative path from the current directory to the path """ - global current_dir, linux + global current_dir + Test.__init__(self, testname) + self.path_to_script = current_dir + path_to_test - if linux: - cmd = ["python3", path_to_test] - else: - cmd = ["py", "-3", path_to_test] + @property + def command(self): + return [sys.executable, self.path_to_script] - Test.__init__(self, testname, cmd, current_dir + path_to_test) + def run_test(self): + log = logdir + os.sep + self.testname + ".log" + progress(self.testname, '>', log, '...') + try: + subprocess_run(self.command, stdout=log) + except FileNotFoundError: + error(red + self.testname + reset + ': executable not found! (' + self.path_to_script + ')') + except subprocess.CalledProcessError as cpe: + if not check_log_for_fails(log, self.testname, self.path_to_script): + # An unexpected error occurred + error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')') class ExeTest(Test): """ @@ -318,7 +318,24 @@ def __init__(self, testname, exe): :param testname: name of the test :param exe: full path to executable """ - Test.__init__(self, testname, exe, exe) + Test.__init__(self, testname) + self.exe = exe + + @property + def command(self): + return [self.exe] + + def run_test(self): + log = logdir + os.sep + self.testname + ".log" + progress(self.testname, '>', log, '...') + try: + subprocess_run(self.command, stdout=log) + except FileNotFoundError: + error(red + self.testname + reset + ': executable not found! (' + self.exe + ')') + except subprocess.CalledProcessError as cpe: + if not check_log_for_fails(log, self.testname, self.exe): + # An unexpected error occurred + error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')') def get_tests(): global regex, target, pyrs, current_dir, linux From b86c0210d6baaaf1ffc5f068526b85fe3312ffbe Mon Sep 17 00:00:00 2001 From: MMirbach Date: Mon, 11 Jan 2021 13:34:14 +0200 Subject: [PATCH 6/7] some small fixes following code review --- unit-tests/run-unit-tests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/unit-tests/run-unit-tests.py b/unit-tests/run-unit-tests.py index 4bb3e151b6..7e68586c5d 100644 --- a/unit-tests/run-unit-tests.py +++ b/unit-tests/run-unit-tests.py @@ -104,7 +104,7 @@ def is_executable(path_to_test): # Parse command-line: try: opts,args = getopt.getopt( sys.argv[1:], 'hvqr:', - longopts = [ 'help', 'verbose', 'debug', 'quiet', 'regex' ]) + longopts = [ 'help', 'verbose', 'debug', 'quiet', 'regex=' ]) except getopt.GetoptError as err: error( err ) # something like "option -a not recognized" usage() @@ -271,7 +271,7 @@ def check_log_for_fails(log, testname, exe): # definition of classes for tests class Test(ABC): # Abstract Base Class """ - Abstract class for a test. Holds the name of the test, the log file for it and the command line needed to run it + Abstract class for a test. Holds the name of the test """ def __init__(self, testname): self.testname = testname @@ -282,7 +282,7 @@ def run_test(self): class PyTest(Test): """ - Class for python tests + Class for python tests. Hold the path to the script of the test """ def __init__(self, testname, path_to_test): """ @@ -298,6 +298,8 @@ def command(self): return [sys.executable, self.path_to_script] def run_test(self): + global n_tests + n_tests +=1 log = logdir + os.sep + self.testname + ".log" progress(self.testname, '>', log, '...') try: @@ -311,7 +313,7 @@ def run_test(self): class ExeTest(Test): """ - Class for c/cpp tests + Class for c/cpp tests. Hold the path to the executable for the test """ def __init__(self, testname, exe): """ @@ -326,6 +328,8 @@ def command(self): return [self.exe] def run_test(self): + global n_tests + n_tests += 1 log = logdir + os.sep + self.testname + ".log" progress(self.testname, '>', log, '...') try: From e9989fb0a94f45e335d1107644961d4e2e2f82c5 Mon Sep 17 00:00:00 2001 From: MMirbach Date: Mon, 11 Jan 2021 15:26:41 +0200 Subject: [PATCH 7/7] fixed bug with getopt --- unit-tests/run-unit-tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/unit-tests/run-unit-tests.py b/unit-tests/run-unit-tests.py index 7e68586c5d..0b854fc495 100644 --- a/unit-tests/run-unit-tests.py +++ b/unit-tests/run-unit-tests.py @@ -123,8 +123,9 @@ def debug(*args): elif opt in ('-q','--quiet'): def out(*args): pass - elif opt[0] in ('-r', '--regex'): - regex = opt[1] + elif opt in ('-r', '--regex'): + regex = arg + if len(args) > 1: usage() target = None @@ -291,7 +292,7 @@ def __init__(self, testname, path_to_test): """ global current_dir Test.__init__(self, testname) - self.path_to_script = current_dir + path_to_test + self.path_to_script = current_dir + os.sep + path_to_test @property def command(self):