[colic] Support scancli

This code provides support to use scancli, a faster version
of scancode.

Signed-off-by: Valerio Cosentino <valcos@bitergia.com>

Author: valeriocos

graal/backends/core/analyzers/nomos.py

@@ -48,12 +48,12 @@ def __init__(self, exec_path):
     def analyze(self, **kwargs):
         """Add information about license
 
-        :param file_path: file path
+        :param file_paths: file path
 
         :returns result: dict of the results of the analysis
         """
         result = {'licenses': []}
-        file_path = kwargs['file_path']
+        file_path = kwargs['file_paths']
 
         try:
             msg = subprocess.check_output([self.exec_path, file_path]).decode("utf-8")

graal/backends/core/analyzers/scancode.py

@@ -28,31 +28,55 @@
 from .analyzer import Analyzer
 
 
+SCANCODE_CLI_EXEC = 'etc/scripts/scancli.py'
+CONFIGURE_EXEC = 'configure'
+
+
 class ScanCode(Analyzer):
     """A wrapper for nexB/scancode-toolkit.
 
     This class allows to call scancode-toolkit over a file, parses
     the result of the analysis and returns it as a dict.
 
     :param exec_path: path of the scancode executable
+    :param cli: True, if scancode_cli is used
     """
     version = '0.1.0'
 
-    def __init__(self, exec_path):
+    def __init__(self, exec_path, cli=False):
+        self.cli = cli
         if not GraalRepository.exists(exec_path):
             raise GraalError(cause="executable path %s not valid" % exec_path)
 
         self.exec_path = exec_path
 
+        if cli:
+            exec_path = self.exec_path.replace(SCANCODE_CLI_EXEC, CONFIGURE_EXEC)
+            _ = subprocess.check_output([exec_path, '--help']).decode("utf-8")
+
     def analyze(self, **kwargs):
         """Add information about license
 
-        :param file_path: file path
+        :param file_paths: file paths
+
+        :returns result: dict of the results of the analysis
+        """
+        if not self.cli:
+            result = self.__analyze_scancode(**kwargs)
+        else:
+            result = self.__analyze_scancode_cli(**kwargs)
+
+        return result
+
+    def __analyze_scancode(self, **kwargs):
+        """Add information about license using scancode
+
+        :param file_paths: file paths
 
         :returns result: dict of the results of the analysis
         """
         result = {'licenses': []}
-        file_path = kwargs['file_path']
+        file_path = kwargs['file_paths']
 
         try:
             msg = subprocess.check_output([self.exec_path, '--json-pp', '-', '--license', file_path]).decode("utf-8")
@@ -68,3 +92,48 @@ def analyze(self, **kwargs):
         result['licenses'] = licenses_raw['files'][0]['licenses']
 
         return result
+
+    def __analyze_scancode_cli(self, **kwargs):
+        """Add information about license using scancode-cli
+
+        :param file_paths: file paths
+
+        :returns result: dict of the results of the analysis
+        """
+        result = {'files': []}
+
+        try:
+            cmd_scancli = ['python3', self.exec_path]
+            cmd_scancli.extend(kwargs['file_paths'])
+            msg = subprocess.check_output(cmd_scancli).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            raise GraalError(cause="Scancode failed at %s, %s" % (' '.join(kwargs['file_paths']),
+                                                                  e.output.decode("utf-8")))
+        finally:
+            subprocess._cleanup()
+
+        if not msg:
+            return {'files': []}
+
+        output_content = ''
+        outputs_json = []
+        for line in msg.split('\n'):
+            if line == '':
+                if output_content:
+                    output_json = json.loads(output_content)[1:]
+                    outputs_json.append(output_json)
+                    output_content = ''
+                else:
+                    continue
+            else:
+                output_content += line
+
+        if output_content:
+            output_json = json.loads(output_content)[1:]
+            outputs_json.append(output_json)
+
+        for output_json in outputs_json:
+            file_info = output_json[0]['files'][0]
+            result['files'].append(file_info)
+
+        return result

graal/backends/core/colic.py

@@ -33,9 +33,11 @@
 
 NOMOS = 'nomos'
 SCANCODE = 'scancode'
+SCANCODE_CLI = 'scancode_cli'
 
 CATEGORY_COLIC_NOMOS = 'code_license_' + NOMOS
 CATEGORY_COLIC_SCANCODE = 'code_license_' + SCANCODE
+CATEGORY_COLIC_SCANCODE_CLI = 'code_license_' + SCANCODE_CLI
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +46,7 @@ class CoLic(Graal):
     """CoLic backend.
 
     This class extends the Graal backend. It gathers license information
-    using Nomos
+    using Nomos, Scancode or Scancode-cli
 
     :param uri: URI of the Git repository
     :param git_path: path to the repository or to the log file
@@ -59,9 +61,9 @@ class CoLic(Graal):
     :raises RepositoryError: raised when there was an error cloning or
         updating the repository.
     """
-    version = '0.4.0'
+    version = '0.5.0'
 
-    CATEGORIES = [CATEGORY_COLIC_NOMOS, CATEGORY_COLIC_SCANCODE]
+    CATEGORIES = [CATEGORY_COLIC_NOMOS, CATEGORY_COLIC_SCANCODE, CATEGORY_COLIC_SCANCODE_CLI]
 
     def __init__(self, uri, git_path, exec_path, worktreepath=DEFAULT_WORKTREE_PATH,
                  entrypoint=None, in_paths=None, out_paths=None,
@@ -84,6 +86,8 @@ def fetch(self, category=CATEGORY_COLIC_NOMOS, paths=None,
 
         if category == CATEGORY_COLIC_SCANCODE:
             self.analyzer_kind = SCANCODE
+        elif category == CATEGORY_COLIC_SCANCODE_CLI:
+            self.analyzer_kind = SCANCODE_CLI
         elif category == CATEGORY_COLIC_NOMOS:
             self.analyzer_kind = NOMOS
         else:
@@ -108,6 +112,8 @@ def metadata_category(item):
             return CATEGORY_COLIC_NOMOS
         elif item['analyzer'] == SCANCODE:
             return CATEGORY_COLIC_SCANCODE
+        elif item['analyzer'] == SCANCODE_CLI:
+            return CATEGORY_COLIC_SCANCODE_CLI
         else:
             raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
 
@@ -135,6 +141,7 @@ def _analyze(self, commit):
         :param commit: a Perceval commit item
         """
         analysis = []
+        files_to_process = []
 
         for committed_file in commit['files']:
 
@@ -148,9 +155,23 @@ def _analyze(self, commit):
             if not GraalRepository.exists(local_path):
                 continue
 
-            license_info = self.analyzer.analyze(local_path)
-            license_info.update({'file_path': file_path})
-            analysis.append(license_info)
+            if self.analyzer_kind == NOMOS:
+                license_info = self.analyzer.analyze(local_path)
+                license_info.update({'file_path': file_path})
+                analysis.append(license_info)
+            elif self.analyzer_kind == SCANCODE:
+                license_info = self.analyzer.analyze(local_path)
+                license_info.update({'file_path': file_path})
+                analysis.append(license_info)
+            else:
+                files_to_process.append((file_path, local_path))
+
+        if files_to_process:
+            local_paths = [f[1] for f in files_to_process]
+            analysis = self.analyzer.analyze(local_paths)
+
+            for i in range(len(analysis['files'])):
+                analysis['files'][i]['file_path'] = files_to_process[i][0]
 
         return analysis
 
@@ -176,11 +197,13 @@ class LicenseAnalyzer:
     def __init__(self, exec_path, kind=NOMOS):
         if kind == SCANCODE:
             self.analyzer = ScanCode(exec_path)
+        elif kind == SCANCODE_CLI:
+            self.analyzer = ScanCode(exec_path, cli=True)
         else:
             self.analyzer = Nomos(exec_path)
 
     def analyze(self, file_path):
-        """Analyze the content of a file using Nomos
+        """Analyze the content of a file using Nomos/Scancode
 
         :param file_path: file path
 
@@ -189,7 +212,7 @@ def analyze(self, file_path):
           'licenses': [..]
         }
         """
-        kwargs = {'file_path': file_path}
+        kwargs = {'file_paths': file_path}
         analysis = self.analyzer.analyze(**kwargs)
 
         return analysis