Fix issue #75: Benchmarks silently execute stock version if scikit-learn-intelex is not installed

bench.py

@@ -187,6 +187,8 @@ def parse_args(parser, size=None, loop_types=(),
                         choices=('host', 'cpu', 'gpu', 'none'),
                         help='Execution context device')
 
+    logging.basicConfig(stream=sys.stderr, format='%(levelname)s: %(message)s', level=logging.INFO)
+
     for data in ['X', 'y']:
         for stage in ['train', 'test']:
             parser.add_argument(f'--file-{data}-{stage}',
@@ -207,15 +209,14 @@ def parse_args(parser, size=None, loop_types=(),
             patch_sklearn()
         except ImportError:
             logging.info('Failed to import sklearnex.patch_sklearn.'
-                         'Use stock version scikit-learn', file=sys.stderr)
+                         'Use stock version scikit-learn')
             params.device = 'none'
     else:
         if params.device != 'none':
             logging.info(
                 'Device context is not supported for stock scikit-learn.'
                 'Please use --no-intel-optimized=False with'
-                f'--device={params.device} parameter. Fallback to --device=none.',
-                file=sys.stderr)
+                f'--device={params.device} parameter. Fallback to --device=none.')
             params.device = 'none'
 
     # disable finiteness check (default)

runner.py

@@ -285,10 +285,10 @@ class GenerationArgs:
                         logging.info(command)
                         if not args.dummy_run:
                             case = f'{lib},{algorithm} ' + case
-                            stdout, stderr = utils.read_output_from_command(
+                            stdout, stderr, ret_code = utils.read_output_from_command(
                                 command, env=os.environ.copy())
                             stdout, extra_stdout = utils.filter_stdout(stdout)
-                            stderr = utils.filter_stderr(stderr)
+                            stderr = utils.filter_stderr(stderr) + '\n'
 
                             print(stdout, end='\n')
 
@@ -303,11 +303,13 @@ class GenerationArgs:
                                 stderr += f'CASE {case} JSON DECODING ERROR:\n' \
                                     + f'{decoding_exception}\n{stdout}\n'
 
-                            if stderr != '':
-                                if 'daal4py' not in stderr:
-                                    is_successful = False
-                                    logging.warning(
-                                        'Error in benchmark: \n' + stderr)
+                            if ret_code == 1:
+                                is_successful = False
+                                logging.warning('Error in benchmark: \n' +
+                                                stderr)
+                            else:
+                                # print info/warnings in benchmark
+                                print(stderr, end='\n')
 
     json.dump(json_result, args.output_file, indent=4)
     name_result_file = args.output_file.name
@@ -319,8 +321,8 @@ class GenerationArgs:
             + f'--report-file {name_result_file}.xlsx '          \
             + '--generation-config ' + args.report
         logging.info(command)
-        stdout, stderr = utils.read_output_from_command(command)
-        if stderr != '':
+        stdout, stderr, ret_code = utils.read_output_from_command(command)
+        if ret_code == 1:
             logging.warning('Error in report generator: \n' + stderr)
             is_successful = False
 

utils.py

@@ -85,14 +85,14 @@ def find_the_dataset(name: str, folder: str, files: Iterable[str]):
 
 
 def read_output_from_command(command: str,
-                             env: Dict[str, str] = os.environ.copy()) -> Tuple[str, str]:
+                             env: Dict[str, str] = os.environ.copy()) -> Tuple[str, str, int]:
     if "PYTHONPATH" in env:
         env["PYTHONPATH"] += ":" + os.path.dirname(os.path.abspath(__file__))
     else:
         env["PYTHONPATH"] = os.path.dirname(os.path.abspath(__file__))
     res = subprocess.run(command.split(' '), stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, encoding='utf-8', env=env)
-    return res.stdout[:-1], res.stderr[:-1]
+    return res.stdout[:-1], res.stderr[:-1], res.returncode
 
 
 def parse_lscpu_lscl_info(command_output: str) -> Dict[str, str]:
@@ -109,7 +109,7 @@ def get_hw_parameters() -> Dict[str, Union[Dict[str, Any], float]]:
 
     hw_params: Dict[str, Union[Dict[str, str], float]] = {'CPU': {}}
     # get CPU information
-    lscpu_info, _ = read_output_from_command('lscpu')
+    lscpu_info, _, _ = read_output_from_command('lscpu')
     lscpu_info = ' '.join(lscpu_info.split())
     for line in lscpu_info.split('\n'):
         k, v = line.split(": ")[:2]
@@ -118,14 +118,14 @@ def get_hw_parameters() -> Dict[str, Union[Dict[str, Any], float]]:
         cast(Dict[str, str], hw_params['CPU'])[k] = v
 
     # get RAM size
-    mem_info, _ = read_output_from_command('free -b')
+    mem_info, _, _ = read_output_from_command('free -b')
     mem_info = mem_info.split('\n')[1]
     mem_info = ' '.join(mem_info.split())
     hw_params['RAM size[GB]'] = int(mem_info.split(' ')[1]) / 2 ** 30
 
     # get Intel GPU information
     try:
-        lsgpu_info, _ = read_output_from_command(
+        lsgpu_info, _, _ = read_output_from_command(
             'lscl --device-type=gpu --platform-vendor=Intel')
         device_num = 0
         start_idx = lsgpu_info.find('Device ')
@@ -141,7 +141,7 @@ def get_hw_parameters() -> Dict[str, Union[Dict[str, Any], float]]:
 
     # get Nvidia GPU information
     try:
-        gpu_info, _ = read_output_from_command(
+        gpu_info, _, _ = read_output_from_command(
             'nvidia-smi --query-gpu=name,memory.total,driver_version,pstate '
             '--format=csv,noheader')
         gpu_info_arr = gpu_info.split(', ')
@@ -160,13 +160,13 @@ def get_hw_parameters() -> Dict[str, Union[Dict[str, Any], float]]:
 def get_sw_parameters() -> Dict[str, Dict[str, Any]]:
     sw_params = {}
     try:
-        gpu_info, _ = read_output_from_command(
+        gpu_info, _, _ = read_output_from_command(
             'nvidia-smi --query-gpu=name,memory.total,driver_version,pstate '
             '--format=csv,noheader')
         info_arr = gpu_info.split(', ')
         sw_params['GPU_driver'] = {'version': info_arr[2]}
         # alert if GPU is already running any processes
-        gpu_processes, _ = read_output_from_command(
+        gpu_processes, _, _ = read_output_from_command(
             'nvidia-smi --query-compute-apps=name,pid,used_memory '
             '--format=csv,noheader')
         if gpu_processes != '':
@@ -177,7 +177,7 @@ def get_sw_parameters() -> Dict[str, Dict[str, Any]]:
 
     # get python packages info from conda
     try:
-        conda_list, _ = read_output_from_command('conda list --json')
+        conda_list, _, _ = read_output_from_command('conda list --json')
         needed_columns = ['version', 'build_string', 'channel']
         conda_list_json: List[Dict[str, str]] = json.loads(conda_list)
         for pkg in conda_list_json: