Merge branch 'develop' of github.com:fasten-project/quality-analyzer

.bettercodehub.yml

@@ -1 +1,4 @@
-component_depth: 2
+component_depth: 1
+
+languages:
+- python

.github/workflows/python-package.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.9]
 
     steps:
     - uses: actions/checkout@v2

Dockerfile

@@ -1,5 +1,5 @@
 ########################################################################################
-FROM python:3.8-slim-buster
+FROM python:3.9-slim-buster
 ########################################################################################
 LABEL maintainer="Software Improvement Group Research <research@sig.eu>"
 

README.md

@@ -29,27 +29,29 @@ The currently supported forges are "mvn", "debian", and "PyPI".
 The plugin will raise an exception if the `forge` in the message is not supported or empty.
 
 #### Maven
-The default topic to consume: `fasten.RepoCloner.out`
+The default topic to consume: `fasten.SyncJava.out`
 
-An example message:
+An example message produced by the SyncJava plugin, which merges out messages from RepoCloner and JavaCGOpal:
 
 ```json
 {
   "input": {},
   "host": "fasten-repo-cloner-56dcf76495-bn4c2",
   "created_at": 1602739158,
-  "plugin_name": "RepoCloner",
-  "payload": {
-    "repoUrl": "",
-    "date": 1291905586,
-    "forge": "mvn",
-    "groupId": "fasten-project",
-    "artifactId": "fasten",
-    "version": "1.0.0",
-    "sourcesUrl": "http://fasten-project/fasten/fasten-1.0.0-sources.jar",
-    "repoPath": "/mnt/fasten/repos/f/fasten-project/fasten",
-    "repoType": "git",
-    "commitTag": "v1.0.0"
+  "plugin_name": "SyncJava",
+  "fasten.RepoCloner.out" : {
+    "payload": {
+        "repoUrl": "",
+        "date": 1291905586,
+        "forge": "mvn",
+        "groupId": "fasten-project",
+        "artifactId": "fasten",
+        "version": "1.0.0",
+        "sourcesUrl": "http://fasten-project/fasten/fasten-1.0.0-sources.jar",
+        "repoPath": "/mnt/fasten/repos/f/fasten-project/fasten",
+        "repoType": "git",
+        "commitTag": "v1.0.0"
+    }
   }
 }
 ```

entrypoint.py

@@ -23,7 +23,7 @@
 
 plugin_name = 'RapidPlugin'
 plugin_description = 'A FASTEN plug-in to populate risk related metadata for a product.'
-plugin_version = '1.0.0'
+plugin_version = '1.1.0'
 
 
 def get_args_parser():

integration_tests/mocks.py

@@ -14,7 +14,7 @@
 #
 
 from fasten.plugins.kafka import KafkaPlugin
-from rapidplugin.kafka_non_blocking import KafkaPluginNonBlocking
+from fasten.plugins.kafka import KafkaPluginNonBlocking
 
 
 class MockConsumer(KafkaPluginNonBlocking):

integration_tests/test_plugin.py

@@ -17,7 +17,6 @@
 from time import sleep
 from integration_tests.mocks import MockConsumer
 from integration_tests.mocks import MockProducer
-from rapidplugin.tests.sources import fix_sourcePath
 
 
 @pytest.fixture()
@@ -111,6 +110,17 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err,
         "repoType": "hg",
         "commitTag": "1.0.0"
     },
+    {
+        "fasten.RepoCloner.out": {
+            "payload":
+                {
+                    "forge": "debian",
+                    "product": "d1",
+                    "version": "1.0.0",
+                    "sourcePath": "/home/plugin/rapidplugin/tests/resources/debian/d1"
+                }
+        }
+    },
     {
         "forge": "debian",
         "product": "d1",

rapidplugin/analysis/lizard_analyzer.py

@@ -19,7 +19,7 @@
 import lizard
 
 from rapidplugin.domain.package import Package, File, Function
-from rapidplugin.utils.utils import MavenUtils
+from rapidplugin.utils.utils import MavenUtils, KafkaUtils
 
 logger = logging.getLogger(__name__)
 
@@ -45,6 +45,9 @@ def analyze(self, payload):
             m.update(metadata)
             m.update(function.metadata())
             m.update(function.metrics())
+            old_f = m['filename']
+            new_f = KafkaUtils.relativize_filename(old_f)
+            m.update({'filename': new_f})
             out_payloads.append(m)
             logger.debug("callable: {}".format(m) + '\n')
         return out_payloads

rapidplugin/rapid_plugin.py

@@ -15,7 +15,7 @@
 
 import datetime
 from time import sleep
-from rapidplugin.kafka_non_blocking import KafkaPluginNonBlocking
+from fasten.plugins.kafka import KafkaPluginNonBlocking
 from rapidplugin.analysis.lizard_analyzer import LizardAnalyzer
 from rapidplugin.utils.utils import KafkaUtils
 
@@ -104,6 +104,7 @@ def consume(self, record):
         Arguments:
           record (JSON): message from self.consume_topic
         '''
+        record = KafkaUtils.extract_from_sync(record)
         payload = record['payload'] if 'payload' in record else record
         in_payload = KafkaUtils.tailor_input(payload)
         try:

rapidplugin/tests/test_analyzer.py

@@ -101,6 +101,12 @@ def analyzer(sources):
     (pypi_message, 2, 1, 5)
 ]
 
+# List of (payload, filename) pairs
+FUNCTION_FILENAME_DATA = [
+    (mvn_message_with_hg_repo, "m3.java"),
+    (debian_message, "d1.c"),
+    (pypi_message, "p1.py")
+]
 
 @pytest.mark.parametrize('record,fc', FUNCTION_COUNT_DATA)
 def test_function_count(analyzer, sources, record, fc: int):
@@ -123,3 +129,10 @@ def test_function_metrics(analyzer, sources, record, nloc: int, complexity: int,
     assert metrics['nloc'] == nloc
     assert metrics['complexity'] == complexity
     assert metrics['token_count'] == token_count
+
+
+@pytest.mark.parametrize('record,filename', FUNCTION_FILENAME_DATA)
+def test_function_filename(analyzer, sources, record, filename):
+    out_payloads = analyzer.analyze(fix_sourcePath(record, sources))
+    metadata = out_payloads[0]
+    assert metadata['filename'] == filename

rapidplugin/tests/test_utils.py

@@ -59,13 +59,13 @@ def test_checkout_git(repo_path, repo_type, commit_tag, sources_dir, repos):
     repo.git.add(".")
     repo.git.commit(m="first commit.")
     repo.create_tag('1.0.0')
-    with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path:
+    with MavenUtils.checkout_version(sources_dir,repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag) as source_path:
         assert sorted(os.listdir(Path(source_path))) == sorted(['1.0.0.zip', 'm1.java'])
 
 @pytest.mark.parametrize('repo_path,repo_type,commit_tag', [("maven/hg/m3", "hg", "1.0.0")])
 def test_checkout_hg(repo_path, repo_type, commit_tag, sources_dir, repos):
     repo_path = os.path.join(repos, repo_path)
-    with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path:
+    with MavenUtils.checkout_version(sources_dir, repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag) as source_path:
         assert sorted(os.listdir(source_path)) == sorted(['m3.java', '.hg_archival.txt'])
 
 @pytest.mark.parametrize('repo_path,repo_type,commit_tag',
@@ -78,7 +78,8 @@ def test_checkout_hg(repo_path, repo_type, commit_tag, sources_dir, repos):
 def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir, repos):
     repo_path = os.path.join(repos, repo_path)
     with pytest.raises(Exception) as e:
-        MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir)
+        MavenUtils.checkout_version(sources_dir, repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag)
+    print(str(e))
 
 PAYLOAD_TAILOR_DATA = [
     ({"product": "a"}, {"product": "a"}),
@@ -91,3 +92,9 @@ def test_tailor_input(in_payload, out_payload):
     tailored = KafkaUtils.tailor_input(in_payload)
     assert tailored == out_payload
 
+@pytest.mark.parametrize('old, new', [
+    ('/private/var/folders/wr/qmdcbfsj4v98v8rwb11zjr5c0000gn/T/pytest-of-cgao/pytest-7/sources0/tmppsmmokh_/d1.c', 'd1.c')
+])
+def test_relativize_filename(old, new):
+    new_file_name = KafkaUtils.relativize_filename(old)
+    assert new_file_name == new

rapidplugin/utils/utils.py

@@ -22,6 +22,7 @@
 from tempfile import TemporaryDirectory
 import os
 import shutil
+import re
 
 
 class MavenUtils:
@@ -48,6 +49,7 @@ def get_source_path(payload, base_dir):
 
     @staticmethod
     def get_source_mvn(payload, base_dir):
+        source_config = {}
         if 'sourcesUrl' in payload:
             sources_url = payload['sourcesUrl']
             if sources_url != "":
@@ -57,7 +59,7 @@ def get_source_mvn(payload, base_dir):
                 repo_path = payload['repoPath']
                 repo_type = payload['repoType']
                 commit_tag = payload['commitTag']
-                source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir)
+                source_path = MavenUtils.checkout_version(base_dir, repo_path=repo_path, repo_type=repo_type, version_tag=commit_tag)
                 return source_path
 
     @staticmethod
@@ -90,22 +92,28 @@ def download_jar(url, base_dir):
         return tmp
 
     @staticmethod
-    def checkout_version(repo_path, repo_type, version_tag, base_dir):
+    def checkout_version(base_dir, **source_config):
+        repo_type = source_config['repo_type']
+        repo_path = source_config['repo_path']
+        version_tag = source_config['version_tag']
         assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type)
         assert repo_path != "", "Empty repo_path."
         assert version_tag != "", "Empty version_tag."
         tmp = TemporaryDirectory(dir=base_dir)
         tmp_path = Path(tmp.name)
         if repo_type == "git":
-            MavenUtils.git_checkout(repo_path, version_tag, tmp_path)
+            MavenUtils.git_checkout(repo_path=repo_path, version_tag=version_tag, tmp_path=tmp_path)
         elif repo_type == "svn":
-            MavenUtils.svn_checkout(repo_path, version_tag, tmp_path)
+            MavenUtils.svn_checkout(repo_path=repo_path, version_tag=version_tag, tmp_path=tmp_path)
         elif repo_type == "hg":
-            MavenUtils.hg_checkout(repo_path, version_tag, tmp_path)
+            MavenUtils.hg_checkout(repo_path=repo_path, version_tag=version_tag, tmp_path=tmp_path)
         return tmp
 
     @staticmethod
-    def git_checkout(repo_path, version_tag, tmp_path):
+    def git_checkout(**source_config):
+        repo_path = source_config['repo_path']
+        version_tag = source_config['version_tag']
+        tmp_path = source_config['tmp_path']
         repo = Repo(repo_path)
         # assert repo.tag(version_tag) is None, "Tag: '{}' does not exist.".format(version_tag)
         archive_name = version_tag+".zip"
@@ -115,14 +123,17 @@ def git_checkout(repo_path, version_tag, tmp_path):
             zipObj.extractall(tmp_path)
 
     @staticmethod
-    def svn_checkout(repo_path, version_tag, tmp_path):
+    def svn_checkout(**source_config):
         raise NotImplementedError("Svn repo not supported.")
         # 'svn export' does not support tag
         # r = LocalClient(repo_path)
         # r.export(tmp_path, version_tag)
 
     @staticmethod
-    def hg_checkout(repo_path, version_tag, tmp_path):
+    def hg_checkout(**source_config):
+        repo_path = source_config['repo_path']
+        version_tag = source_config['version_tag']
+        tmp_path = source_config['tmp_path']
         wd = os.getcwd()
         os.chdir(repo_path)
         cmd = [
@@ -180,4 +191,27 @@ def tailor_input(payload):
                 payload[key] = tailor[key]
         return payload
 
+    @staticmethod
+    def extract_from_sync(payload):
+        """
+        Extract content of RepoCloner in the synchronized topic.
+        :param payload: payload of fasten.SyncJava.out, see
+                        https://github.com/fasten-project/synchronize-javacg
+        :return: payload of RepoCloner, see
+                        https://github.com/fasten-project/fasten/wiki/Kafka-Topics#fastenrepocloner
+        """
+        extract = payload['fasten.RepoCloner.out'] if 'fasten.RepoCloner.out' in payload else payload
+        return extract
 
+    @staticmethod
+    def relativize_filename(filename):
+        """
+        Extract the relative path of the source code file.
+        :param filename: absolute path included by Lizard tool,
+                        e.g. 'work_directory/tmppsmmokh_/d1.c'
+        :return: filename relative to the temporal source directory,
+                        e.g. 'd1.c'
+
+        """
+        regex = re.compile('(/tmp).{8}/')
+        return regex.split(filename)[2]