From 960730a6bbabde9a3a0ce63b352bb37fa8f3bfdf Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 27 Oct 2020 15:43:45 +0100 Subject: [PATCH 01/32] Added svn and hg check out (without testing) --- rapidplugin/analysis/lizard_analyzer.py | 19 +++--- .../tests/resources/maven/git/m1/m1.java | 4 +- .../tests/resources/maven/svn/m2/m2.java | 5 ++ rapidplugin/utils/utils.py | 60 ++++++++++--------- 4 files changed, 50 insertions(+), 38 deletions(-) create mode 100644 rapidplugin/tests/resources/maven/svn/m2/m2.java diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 8e9bd03..4f0a132 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -60,19 +60,22 @@ def get_source_path(self, payload): [ ] 2.2 if needed, checkout based on the release date. [ ] 3. else return null """ + source_path = None if payload['forge'] == "mvn": if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] - return MavenUtils.download_jar(sources_url, self.base_dir) - else: - if 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: - repo_path = payload['repoPath'] - repo_type = payload['repoType'] - commit_tag = payload['commitTag'] - return MavenUtils.checkout_version(repo_path, repo_type, commit_tag) + source_path = MavenUtils.download_jar(sources_url, self.base_dir) + elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: + repo_path = payload['repoPath'] + repo_type = payload['repoType'] + commit_tag = payload['commitTag'] + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag) + assert source_path is not None, \ + f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." + return source_path else: source_path = payload['sourcePath'] - assert os.path.isabs(source_path), "sourcePath is not an absolute path!" + assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) return source_path def clean_up(self): diff --git a/rapidplugin/tests/resources/maven/git/m1/m1.java b/rapidplugin/tests/resources/maven/git/m1/m1.java index 73451ce..43c9872 100644 --- a/rapidplugin/tests/resources/maven/git/m1/m1.java +++ b/rapidplugin/tests/resources/maven/git/m1/m1.java @@ -1,5 +1,5 @@ -class C { +class M1 { public static void main(String[] args) { - System.out.println("test java!"); + System.out.println("M1 from git!"); } } \ No newline at end of file diff --git a/rapidplugin/tests/resources/maven/svn/m2/m2.java b/rapidplugin/tests/resources/maven/svn/m2/m2.java new file mode 100644 index 0000000..22764e2 --- /dev/null +++ b/rapidplugin/tests/resources/maven/svn/m2/m2.java @@ -0,0 +1,5 @@ +class M2 { + public static void main(String[] args) { + System.out.println("M2 from svn!"); + } +} \ No newline at end of file diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 875bf68..e412797 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -15,29 +15,27 @@ from zipfile import ZipFile from pathlib import Path -from git import Repo, GitCommandError +from git import Repo +from svn.local import LocalClient import requests -import os +import subprocess as sp class MavenUtils: - # TODO:add exception handling @staticmethod def download_jar(url, base_dir): - if url == "": - return "" - else: - base_dir = Path(base_dir) - if not base_dir.exists(): - base_dir.mkdir(parents=True) - file_name = base_dir/url.split('/')[-1] - tmp_dir = base_dir/"tmp" - r = requests.get(url, allow_redirects=True) - open(file_name, 'wb').write(r.content) - with ZipFile(file_name, 'r') as zipObj: - zipObj.extractall(tmp_dir) - return tmp_dir + assert url != "", "Invalid URL for 'sources.jar'." + base_dir = Path(base_dir) + if not base_dir.exists(): + base_dir.mkdir(parents=True) + file_name = base_dir/url.split('/')[-1] + tmp_dir = base_dir/"tmp" + r = requests.get(url, allow_redirects=True) + open(file_name, 'wb').write(r.content) + with ZipFile(file_name, 'r') as zipObj: + zipObj.extractall(tmp_dir) + return tmp_dir @staticmethod def checkout_version(repo_path, repo_type, version_tag, base_dir): @@ -45,23 +43,29 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): if not base_dir.exists(): base_dir.mkdir(parents=True) tmp_dir = base_dir/"tmp" + assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) if repo_type == "git": repo = Repo(repo_path) assert repo.tags[version_tag] is not None archive_name = version_tag+".zip" archive_file_name = tmp_dir/archive_name - try: - # or use repo.archive() - repo.git.archive(version_tag, o=archive_file_name) - with ZipFile(archive_file_name, 'r') as zipObj: - zipObj.extractall(tmp_dir) - return tmp_dir - except GitCommandError: - return "" - if repo_type == "svn": - return "" - if repo_type == "hg": - return "" + repo.git.archive(version_tag, o=archive_file_name) + with ZipFile(archive_file_name, 'r') as zipObj: + zipObj.extractall(tmp_dir) + elif repo_type == "svn": + r = LocalClient(repo_path) + r.export(tmp_dir, version_tag) + elif repo_type == "hg": + cmd = [ + 'hg', + 'archive', + '-r', version_tag, + '-t', 'files', + tmp_dir + ] + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + o, e = proc.communicate() + return tmp_dir class KafkaUtils: From 7ab5d7076e78aec55d5679292b9bc4c6073258d6 Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 27 Oct 2020 16:09:46 +0100 Subject: [PATCH 02/32] Reworked assertions for various situations in the analysis. --- rapidplugin/analysis/lizard_analyzer.py | 6 +++++- rapidplugin/utils/utils.py | 24 +++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 4f0a132..ad6b133 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -69,12 +69,16 @@ def get_source_path(self, payload): repo_path = payload['repoPath'] repo_type = payload['repoType'] commit_tag = payload['commitTag'] - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag) + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, self.base_dir) assert source_path is not None, \ f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." return source_path else: + assert 'sourcePath' in payload, \ + f"Cannot get source code for '{payload['product']}:{payload['version']}', missing 'sourcePath'." source_path = payload['sourcePath'] + assert source_path != "", \ + f"Cannot get source code for '{payload['product']}:{payload['version']}', empty 'sourcePath." assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) return source_path diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index e412797..802ba1c 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -53,18 +53,20 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): with ZipFile(archive_file_name, 'r') as zipObj: zipObj.extractall(tmp_dir) elif repo_type == "svn": - r = LocalClient(repo_path) - r.export(tmp_dir, version_tag) + return None + # r = LocalClient(repo_path) + # r.export(tmp_dir, version_tag) elif repo_type == "hg": - cmd = [ - 'hg', - 'archive', - '-r', version_tag, - '-t', 'files', - tmp_dir - ] - proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - o, e = proc.communicate() + return None + # cmd = [ + # 'hg', + # 'archive', + # '-r', version_tag, + # '-t', 'files', + # tmp_dir + # ] + # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + # o, e = proc.communicate() return tmp_dir From 6303216dfa0ed14a323ffc8a3a849df25d2ce5b5 Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 27 Oct 2020 17:20:26 +0100 Subject: [PATCH 03/32] Reworked assertions for various situations in the analysis(not finish) without testing. --- README.md | 62 +++++++++++++++++++++++++ rapidplugin/analysis/lizard_analyzer.py | 3 +- rapidplugin/utils/utils.py | 3 +- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3759bea..7ef2b5b 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,65 @@ # RAPID RAPID is the quality analysis application that SIG developes for the FASTEN project in WP3. + +# Components + +## RapidPlugin + +RapidPlugin is a FASTEN plugin that +generates code complexity data for the `product`. + +The plugin consumes messages in the Kafka topics, +generates code complexity data using `Lizard`, and +produces Kafka topics with complexity data at the `callable` level. + +### Input Kafka topics + +#### Maven +`fasten.RepoCloner.out` + +An example message: + +```json +{ + "input": { + ... + }, + "host": "fasten-repo-cloner-56dcf76495-bn4c2", + "created_at": 1602739158, + "plugin_name": "RepoCloner", + "payload": { + "repoUrl": "", + "date": 1291905586, + "forge": "mvn", + "groupId": "fasten-project", + "artifactId": "fasten", + "version": "1.0.0", + "sourcesUrl": "http://fasten-project/fasten/fasten-1.0.0-sources.jar", + "repoPath": "/mnt/fasten/repos/f/fasten-project/fasten", + "repoType": "git", + "commitTag": "v1.0.0" + } +} +``` +The message should have all the information to identify a unique `product`. +For `Maven`, the fields `forge`, `groupId`, `artifactId`, and `version` +should not be empty. +Missing any of these fields will cause exceptions in message consuming. + +The message should have at least one way to point out the source to the code. +If `sourcesUrl` presents and is non-empty, the plugin will download the source code. + + + +#### PyPI + +#### Debian + +### Output Kafka topics + +#### Output topic + +#### Log topic + +#### Error topic diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index ad6b133..45b717f 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -64,7 +64,8 @@ def get_source_path(self, payload): if payload['forge'] == "mvn": if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] - source_path = MavenUtils.download_jar(sources_url, self.base_dir) + if sources_url != "": + source_path = MavenUtils.download_jar(sources_url, self.base_dir) elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: repo_path = payload['repoPath'] repo_type = payload['repoType'] diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 802ba1c..0368d8b 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -25,7 +25,6 @@ class MavenUtils: @staticmethod def download_jar(url, base_dir): - assert url != "", "Invalid URL for 'sources.jar'." base_dir = Path(base_dir) if not base_dir.exists(): base_dir.mkdir(parents=True) @@ -44,6 +43,8 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): base_dir.mkdir(parents=True) tmp_dir = base_dir/"tmp" assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) + assert repo_path != "", "Empty repo_path." + assert version_tag != "", "Empty version_tag." if repo_type == "git": repo = Repo(repo_path) assert repo.tags[version_tag] is not None From a78c5b0a94954c03335c811185608e38823f120c Mon Sep 17 00:00:00 2001 From: cg122 Date: Wed, 28 Oct 2020 12:08:47 +0100 Subject: [PATCH 04/32] Updated documentation on supported message in consuming. --- README.md | 155 ++++++++++++++++++++++-- rapidplugin/analysis/lizard_analyzer.py | 18 ++- rapidplugin/utils/utils.py | 32 +++-- 3 files changed, 179 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 7ef2b5b..7ae0284 100644 --- a/README.md +++ b/README.md @@ -18,16 +18,19 @@ produces Kafka topics with complexity data at the `callable` level. ### Input Kafka topics +The plugin will trigger different pipelines based on the `forge`, +so the field `forge` is a mandatory in any incoming messages. +The currently supported forges are "mvn", "debian", and "PyPI". +The plugin will raise an exception if the `forge` in the message is not supported or empty. + #### Maven -`fasten.RepoCloner.out` +The default topic to consume: `fasten.RepoCloner.out` An example message: ```json { - "input": { - ... - }, + "input": {}, "host": "fasten-repo-cloner-56dcf76495-bn4c2", "created_at": 1602739158, "plugin_name": "RepoCloner", @@ -46,23 +49,161 @@ An example message: } ``` The message should have all the information to identify a unique `product`. -For `Maven`, the fields `forge`, `groupId`, `artifactId`, and `version` +For **Maven**, the fields `groupId`, `artifactId`, and `version` should not be empty. Missing any of these fields will cause exceptions in message consuming. -The message should have at least one way to point out the source to the code. -If `sourcesUrl` presents and is non-empty, the plugin will download the source code. +The message should have at least one way to point out the link to the source code. +- `sourcesUrl` + +This field is the most reliable pointer to the versioned source code of a `product`. + +If `sourcesUrl` presents and is non-empty, the plugin will download the source code from the url specified in `sourcesUrl`. + +If `sourcesUrl` does not present or is empty, the plugin will try the other sources to get the source code. +- `repoPath` +If `repoPath` is not empty, the repository of the `product` has been cloned to the FASTEN server. +The plugin will try to check out the right version of the source code if both `repoType` and `commitTag` are non-empty. +The currently supported`repoType` are "git", "svn", and "hg". +The plugin will raise an exception if the `repoType` in the message is not supported. + +If none of the above efforts succeed, the plugin will raise an exception +specifying that it cannot get the source code. #### PyPI +The current topic to consume: `fasten.pycg.with_sources.out` + +An example message: + +```json +{ + "input": {}, + "plugin_name": "PyCG", + "plugin_version": "0.0.1", + "created_at": "1596576724", + "payload": { + "product": "gud", + "forge": "PyPI", + "generator": "PyCG", + "depset": [], + "version": "1.0.10", + "timestamp": "1561421985", + "modules": { + }, + "graph":{}, + "cha": {}, + "metadata": {}, + "sourcePath": "/mnt/fasten/pypi-test/final/sources/g/gud/1.0.10" + } +} +``` + +The message should have all the information to identify a unique `product`. +For **PyPI**, the fields `product`, and `version` should not be empty. +Missing any of these fields will cause exceptions in message consuming. + +If `sourcePath` is empty, the plugin will raise an exception. #### Debian +The current topic to consume: `fasten.debian.cg.2` + +An example message: + +```json +{ + "plugin_name": "CScoutKafkaPlugin", + "plugin_version": "0.0.1", + "input": { + "package": "sed", + "version": "4.7-1", + "arch": "amd64", + "release": "buster", + "source": "sed", + "source_version": "4.7-1", + "date": "" + }, + "created_at": "1600861444.064117", + "payload": { + "forge": "debian", + "release": "", + "product": "sed", + "version": "4.7-1", + "source": "sed", + "architecture": "amd64", + "generator": "cscout", + "timestamp": "1545470644", + "depset": [], + "build_depset": [], + "undeclared_depset": [], + "graph": {}, + "functions": {}, + "profiling_data": {}, + "sourcePath": "/mnt/fasten/debian/sources/s/sed/4.7-1" + } +} +``` +Similar to **PyPI**, the fields `product`, and `version` should not be empty. +Missing any of these fields will cause exceptions in message consuming. + +If `sourcePath` is empty, the plugin will raise an exception. ### Output Kafka topics +The field `input` in the output topic is used for tracking upstreaming plugins and +usually copies the whole `payload` from the consumed message. +To avoid potential large message in the output topics, +the plugin will tailor the `payload` of consumed message. +The content of the following fields will be tailored. + +**PyPI**: `fasten.pycg.with_sources.out` +- `depset` +- `cha` +- `graph` +- `modules` + +**Debian**: `fasten.debian.cg.2` +- `depset` +- `build_depset` +- `undeclared_depset` +- `graph` +- `functions` + #### Output topic +The default topic to produce: `fasten.RapidPlugin.callable.out` + +An example message: +```json +{ + "plugin_name": "RapidPlugin", + "plugin_version": "0.0.1", + "input": {}, + "created_at": "1595434993", + "payload": { + "analyzer_name": "Lizard", + "analyzer_version": "1.17.7", + "analysis_timestamp": "1596455923", + "product": "fasten-project:fasten", + "version": "1.0.0", + "forge": "mvn", + "language": "java", + "filepath": "/fasten/core/server.java", + "name": "callable", + "long_name": "callable(int i)", + "start_line": 33, + "end_line": 42, + "metrics": { + "nloc": 10, + "complexity": 5, + "...": 0 + } + } +} +``` #### Log topic +The default topic to produce: `fasten.RapidPlugin.callable.log` #### Error topic +The default topic to produce: `fasten.RapidPlugin.callable.err` diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 45b717f..48c5401 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -16,6 +16,8 @@ import os import logging import datetime +from pathlib import Path + import lizard from rapidplugin.domain.package import Package, File, Function @@ -58,19 +60,22 @@ def get_source_path(self, payload): [x] 2. else if repoPath is not empty, and [x] 2.1 if commit tag is valid, checkout based on tag and return the path [ ] 2.2 if needed, checkout based on the release date. - [ ] 3. else return null + [x] 3. else return None and raise exception (Cannot get source code) """ + base_dir = Path(self.base_dir) + if not base_dir.exists(): + base_dir.mkdir(parents=True) source_path = None if payload['forge'] == "mvn": if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] if sources_url != "": - source_path = MavenUtils.download_jar(sources_url, self.base_dir) + source_path = MavenUtils.download_jar(sources_url, base_dir) elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: repo_path = payload['repoPath'] repo_type = payload['repoType'] commit_tag = payload['commitTag'] - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, self.base_dir) + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) assert source_path is not None, \ f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." return source_path @@ -83,13 +88,6 @@ def get_source_path(self, payload): assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) return source_path - def clean_up(self): - ''' - TODO - ''' - # if os.path.exists(self.base_dir): - # shutil.rmtree(self.base_dir) - class LizardPackage(Package): diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 0368d8b..5f159ac 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -25,11 +25,8 @@ class MavenUtils: @staticmethod def download_jar(url, base_dir): - base_dir = Path(base_dir) - if not base_dir.exists(): - base_dir.mkdir(parents=True) + tmp_dir = base_dir / "tmp" file_name = base_dir/url.split('/')[-1] - tmp_dir = base_dir/"tmp" r = requests.get(url, allow_redirects=True) open(file_name, 'wb').write(r.content) with ZipFile(file_name, 'r') as zipObj: @@ -38,16 +35,13 @@ def download_jar(url, base_dir): @staticmethod def checkout_version(repo_path, repo_type, version_tag, base_dir): - base_dir = Path(base_dir) - if not base_dir.exists(): - base_dir.mkdir(parents=True) - tmp_dir = base_dir/"tmp" + tmp_dir = base_dir / "tmp" assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) assert repo_path != "", "Empty repo_path." assert version_tag != "", "Empty version_tag." if repo_type == "git": repo = Repo(repo_path) - assert repo.tags[version_tag] is not None + assert repo.tags[version_tag] is not None, "Tag: '{}' does not exist.".format(version_tag) archive_name = version_tag+".zip" archive_file_name = tmp_dir/archive_name repo.git.archive(version_tag, o=archive_file_name) @@ -101,12 +95,32 @@ def tailor_input(payload): cha = { "cha": {} } + depset = { + "depset": [] + } + build_depset = { + "build_depset": [] + } + undeclared_depset = { + "undeclared_depset": [] + } + functions = { + "functions" :{} + } if 'graph' in payload: payload.update(graph) if 'modules' in payload: payload.update(modules) if 'cha' in payload: payload.update(cha) + if 'depset' in payload: + payload.update(depset) + if 'build_depset' in payload: + payload.update(build_depset) + if 'undeclared_depset' in payload: + payload.update(undeclared_depset) + if 'functions' in payload: + payload.update(functions) return payload From b72c6771f01c305eb93cac43347d0bb8f9075d03 Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 27 Oct 2020 15:43:45 +0100 Subject: [PATCH 05/32] Added svn and hg check out (without testing) --- rapidplugin/analysis/lizard_analyzer.py | 19 +++--- .../tests/resources/maven/git/m1/m1.java | 4 +- .../tests/resources/maven/svn/m2/m2.java | 5 ++ rapidplugin/utils/utils.py | 60 ++++++++++--------- 4 files changed, 50 insertions(+), 38 deletions(-) create mode 100644 rapidplugin/tests/resources/maven/svn/m2/m2.java diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 8e9bd03..4f0a132 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -60,19 +60,22 @@ def get_source_path(self, payload): [ ] 2.2 if needed, checkout based on the release date. [ ] 3. else return null """ + source_path = None if payload['forge'] == "mvn": if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] - return MavenUtils.download_jar(sources_url, self.base_dir) - else: - if 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: - repo_path = payload['repoPath'] - repo_type = payload['repoType'] - commit_tag = payload['commitTag'] - return MavenUtils.checkout_version(repo_path, repo_type, commit_tag) + source_path = MavenUtils.download_jar(sources_url, self.base_dir) + elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: + repo_path = payload['repoPath'] + repo_type = payload['repoType'] + commit_tag = payload['commitTag'] + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag) + assert source_path is not None, \ + f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." + return source_path else: source_path = payload['sourcePath'] - assert os.path.isabs(source_path), "sourcePath is not an absolute path!" + assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) return source_path def clean_up(self): diff --git a/rapidplugin/tests/resources/maven/git/m1/m1.java b/rapidplugin/tests/resources/maven/git/m1/m1.java index 73451ce..43c9872 100644 --- a/rapidplugin/tests/resources/maven/git/m1/m1.java +++ b/rapidplugin/tests/resources/maven/git/m1/m1.java @@ -1,5 +1,5 @@ -class C { +class M1 { public static void main(String[] args) { - System.out.println("test java!"); + System.out.println("M1 from git!"); } } \ No newline at end of file diff --git a/rapidplugin/tests/resources/maven/svn/m2/m2.java b/rapidplugin/tests/resources/maven/svn/m2/m2.java new file mode 100644 index 0000000..22764e2 --- /dev/null +++ b/rapidplugin/tests/resources/maven/svn/m2/m2.java @@ -0,0 +1,5 @@ +class M2 { + public static void main(String[] args) { + System.out.println("M2 from svn!"); + } +} \ No newline at end of file diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 875bf68..e412797 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -15,29 +15,27 @@ from zipfile import ZipFile from pathlib import Path -from git import Repo, GitCommandError +from git import Repo +from svn.local import LocalClient import requests -import os +import subprocess as sp class MavenUtils: - # TODO:add exception handling @staticmethod def download_jar(url, base_dir): - if url == "": - return "" - else: - base_dir = Path(base_dir) - if not base_dir.exists(): - base_dir.mkdir(parents=True) - file_name = base_dir/url.split('/')[-1] - tmp_dir = base_dir/"tmp" - r = requests.get(url, allow_redirects=True) - open(file_name, 'wb').write(r.content) - with ZipFile(file_name, 'r') as zipObj: - zipObj.extractall(tmp_dir) - return tmp_dir + assert url != "", "Invalid URL for 'sources.jar'." + base_dir = Path(base_dir) + if not base_dir.exists(): + base_dir.mkdir(parents=True) + file_name = base_dir/url.split('/')[-1] + tmp_dir = base_dir/"tmp" + r = requests.get(url, allow_redirects=True) + open(file_name, 'wb').write(r.content) + with ZipFile(file_name, 'r') as zipObj: + zipObj.extractall(tmp_dir) + return tmp_dir @staticmethod def checkout_version(repo_path, repo_type, version_tag, base_dir): @@ -45,23 +43,29 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): if not base_dir.exists(): base_dir.mkdir(parents=True) tmp_dir = base_dir/"tmp" + assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) if repo_type == "git": repo = Repo(repo_path) assert repo.tags[version_tag] is not None archive_name = version_tag+".zip" archive_file_name = tmp_dir/archive_name - try: - # or use repo.archive() - repo.git.archive(version_tag, o=archive_file_name) - with ZipFile(archive_file_name, 'r') as zipObj: - zipObj.extractall(tmp_dir) - return tmp_dir - except GitCommandError: - return "" - if repo_type == "svn": - return "" - if repo_type == "hg": - return "" + repo.git.archive(version_tag, o=archive_file_name) + with ZipFile(archive_file_name, 'r') as zipObj: + zipObj.extractall(tmp_dir) + elif repo_type == "svn": + r = LocalClient(repo_path) + r.export(tmp_dir, version_tag) + elif repo_type == "hg": + cmd = [ + 'hg', + 'archive', + '-r', version_tag, + '-t', 'files', + tmp_dir + ] + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + o, e = proc.communicate() + return tmp_dir class KafkaUtils: From 516e876f7f4a321c606a1b26722874047e4d73cb Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 27 Oct 2020 16:09:46 +0100 Subject: [PATCH 06/32] Reworked assertions for various situations in the analysis. --- rapidplugin/analysis/lizard_analyzer.py | 6 +++++- rapidplugin/utils/utils.py | 24 +++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 4f0a132..ad6b133 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -69,12 +69,16 @@ def get_source_path(self, payload): repo_path = payload['repoPath'] repo_type = payload['repoType'] commit_tag = payload['commitTag'] - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag) + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, self.base_dir) assert source_path is not None, \ f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." return source_path else: + assert 'sourcePath' in payload, \ + f"Cannot get source code for '{payload['product']}:{payload['version']}', missing 'sourcePath'." source_path = payload['sourcePath'] + assert source_path != "", \ + f"Cannot get source code for '{payload['product']}:{payload['version']}', empty 'sourcePath." assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) return source_path diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index e412797..802ba1c 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -53,18 +53,20 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): with ZipFile(archive_file_name, 'r') as zipObj: zipObj.extractall(tmp_dir) elif repo_type == "svn": - r = LocalClient(repo_path) - r.export(tmp_dir, version_tag) + return None + # r = LocalClient(repo_path) + # r.export(tmp_dir, version_tag) elif repo_type == "hg": - cmd = [ - 'hg', - 'archive', - '-r', version_tag, - '-t', 'files', - tmp_dir - ] - proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - o, e = proc.communicate() + return None + # cmd = [ + # 'hg', + # 'archive', + # '-r', version_tag, + # '-t', 'files', + # tmp_dir + # ] + # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + # o, e = proc.communicate() return tmp_dir From cf0c59ff1e2767959c2cab667bfe08cfd8c1dc91 Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 27 Oct 2020 17:20:26 +0100 Subject: [PATCH 07/32] Reworked assertions for various situations in the analysis(not finish) without testing. --- README.md | 62 +++++++++++++++++++++++++ rapidplugin/analysis/lizard_analyzer.py | 3 +- rapidplugin/utils/utils.py | 3 +- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be0f9c2..e10b8f8 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,65 @@ # RAPID RAPID is the quality analysis application that SIG developes for the FASTEN project in WP3. + +# Components + +## RapidPlugin + +RapidPlugin is a FASTEN plugin that +generates code complexity data for the `product`. + +The plugin consumes messages in the Kafka topics, +generates code complexity data using `Lizard`, and +produces Kafka topics with complexity data at the `callable` level. + +### Input Kafka topics + +#### Maven +`fasten.RepoCloner.out` + +An example message: + +```json +{ + "input": { + ... + }, + "host": "fasten-repo-cloner-56dcf76495-bn4c2", + "created_at": 1602739158, + "plugin_name": "RepoCloner", + "payload": { + "repoUrl": "", + "date": 1291905586, + "forge": "mvn", + "groupId": "fasten-project", + "artifactId": "fasten", + "version": "1.0.0", + "sourcesUrl": "http://fasten-project/fasten/fasten-1.0.0-sources.jar", + "repoPath": "/mnt/fasten/repos/f/fasten-project/fasten", + "repoType": "git", + "commitTag": "v1.0.0" + } +} +``` +The message should have all the information to identify a unique `product`. +For `Maven`, the fields `forge`, `groupId`, `artifactId`, and `version` +should not be empty. +Missing any of these fields will cause exceptions in message consuming. + +The message should have at least one way to point out the source to the code. +If `sourcesUrl` presents and is non-empty, the plugin will download the source code. + + + +#### PyPI + +#### Debian + +### Output Kafka topics + +#### Output topic + +#### Log topic + +#### Error topic diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index ad6b133..45b717f 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -64,7 +64,8 @@ def get_source_path(self, payload): if payload['forge'] == "mvn": if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] - source_path = MavenUtils.download_jar(sources_url, self.base_dir) + if sources_url != "": + source_path = MavenUtils.download_jar(sources_url, self.base_dir) elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: repo_path = payload['repoPath'] repo_type = payload['repoType'] diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 802ba1c..0368d8b 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -25,7 +25,6 @@ class MavenUtils: @staticmethod def download_jar(url, base_dir): - assert url != "", "Invalid URL for 'sources.jar'." base_dir = Path(base_dir) if not base_dir.exists(): base_dir.mkdir(parents=True) @@ -44,6 +43,8 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): base_dir.mkdir(parents=True) tmp_dir = base_dir/"tmp" assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) + assert repo_path != "", "Empty repo_path." + assert version_tag != "", "Empty version_tag." if repo_type == "git": repo = Repo(repo_path) assert repo.tags[version_tag] is not None From fca852d0e1397c6feaaeb5eff89a2f1e63af1d1e Mon Sep 17 00:00:00 2001 From: cg122 Date: Wed, 28 Oct 2020 12:08:47 +0100 Subject: [PATCH 08/32] Updated documentation on supported message in consuming. --- README.md | 155 ++++++++++++++++++++++-- rapidplugin/analysis/lizard_analyzer.py | 18 ++- rapidplugin/utils/utils.py | 32 +++-- 3 files changed, 179 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index e10b8f8..1d6af29 100644 --- a/README.md +++ b/README.md @@ -23,16 +23,19 @@ produces Kafka topics with complexity data at the `callable` level. ### Input Kafka topics +The plugin will trigger different pipelines based on the `forge`, +so the field `forge` is a mandatory in any incoming messages. +The currently supported forges are "mvn", "debian", and "PyPI". +The plugin will raise an exception if the `forge` in the message is not supported or empty. + #### Maven -`fasten.RepoCloner.out` +The default topic to consume: `fasten.RepoCloner.out` An example message: ```json { - "input": { - ... - }, + "input": {}, "host": "fasten-repo-cloner-56dcf76495-bn4c2", "created_at": 1602739158, "plugin_name": "RepoCloner", @@ -51,23 +54,161 @@ An example message: } ``` The message should have all the information to identify a unique `product`. -For `Maven`, the fields `forge`, `groupId`, `artifactId`, and `version` +For **Maven**, the fields `groupId`, `artifactId`, and `version` should not be empty. Missing any of these fields will cause exceptions in message consuming. -The message should have at least one way to point out the source to the code. -If `sourcesUrl` presents and is non-empty, the plugin will download the source code. +The message should have at least one way to point out the link to the source code. +- `sourcesUrl` + +This field is the most reliable pointer to the versioned source code of a `product`. + +If `sourcesUrl` presents and is non-empty, the plugin will download the source code from the url specified in `sourcesUrl`. + +If `sourcesUrl` does not present or is empty, the plugin will try the other sources to get the source code. +- `repoPath` +If `repoPath` is not empty, the repository of the `product` has been cloned to the FASTEN server. +The plugin will try to check out the right version of the source code if both `repoType` and `commitTag` are non-empty. +The currently supported`repoType` are "git", "svn", and "hg". +The plugin will raise an exception if the `repoType` in the message is not supported. + +If none of the above efforts succeed, the plugin will raise an exception +specifying that it cannot get the source code. #### PyPI +The current topic to consume: `fasten.pycg.with_sources.out` + +An example message: + +```json +{ + "input": {}, + "plugin_name": "PyCG", + "plugin_version": "0.0.1", + "created_at": "1596576724", + "payload": { + "product": "gud", + "forge": "PyPI", + "generator": "PyCG", + "depset": [], + "version": "1.0.10", + "timestamp": "1561421985", + "modules": { + }, + "graph":{}, + "cha": {}, + "metadata": {}, + "sourcePath": "/mnt/fasten/pypi-test/final/sources/g/gud/1.0.10" + } +} +``` + +The message should have all the information to identify a unique `product`. +For **PyPI**, the fields `product`, and `version` should not be empty. +Missing any of these fields will cause exceptions in message consuming. + +If `sourcePath` is empty, the plugin will raise an exception. #### Debian +The current topic to consume: `fasten.debian.cg.2` + +An example message: + +```json +{ + "plugin_name": "CScoutKafkaPlugin", + "plugin_version": "0.0.1", + "input": { + "package": "sed", + "version": "4.7-1", + "arch": "amd64", + "release": "buster", + "source": "sed", + "source_version": "4.7-1", + "date": "" + }, + "created_at": "1600861444.064117", + "payload": { + "forge": "debian", + "release": "", + "product": "sed", + "version": "4.7-1", + "source": "sed", + "architecture": "amd64", + "generator": "cscout", + "timestamp": "1545470644", + "depset": [], + "build_depset": [], + "undeclared_depset": [], + "graph": {}, + "functions": {}, + "profiling_data": {}, + "sourcePath": "/mnt/fasten/debian/sources/s/sed/4.7-1" + } +} +``` +Similar to **PyPI**, the fields `product`, and `version` should not be empty. +Missing any of these fields will cause exceptions in message consuming. + +If `sourcePath` is empty, the plugin will raise an exception. ### Output Kafka topics +The field `input` in the output topic is used for tracking upstreaming plugins and +usually copies the whole `payload` from the consumed message. +To avoid potential large message in the output topics, +the plugin will tailor the `payload` of consumed message. +The content of the following fields will be tailored. + +**PyPI**: `fasten.pycg.with_sources.out` +- `depset` +- `cha` +- `graph` +- `modules` + +**Debian**: `fasten.debian.cg.2` +- `depset` +- `build_depset` +- `undeclared_depset` +- `graph` +- `functions` + #### Output topic +The default topic to produce: `fasten.RapidPlugin.callable.out` + +An example message: +```json +{ + "plugin_name": "RapidPlugin", + "plugin_version": "0.0.1", + "input": {}, + "created_at": "1595434993", + "payload": { + "analyzer_name": "Lizard", + "analyzer_version": "1.17.7", + "analysis_timestamp": "1596455923", + "product": "fasten-project:fasten", + "version": "1.0.0", + "forge": "mvn", + "language": "java", + "filepath": "/fasten/core/server.java", + "name": "callable", + "long_name": "callable(int i)", + "start_line": 33, + "end_line": 42, + "metrics": { + "nloc": 10, + "complexity": 5, + "...": 0 + } + } +} +``` #### Log topic +The default topic to produce: `fasten.RapidPlugin.callable.log` #### Error topic +The default topic to produce: `fasten.RapidPlugin.callable.err` diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 45b717f..48c5401 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -16,6 +16,8 @@ import os import logging import datetime +from pathlib import Path + import lizard from rapidplugin.domain.package import Package, File, Function @@ -58,19 +60,22 @@ def get_source_path(self, payload): [x] 2. else if repoPath is not empty, and [x] 2.1 if commit tag is valid, checkout based on tag and return the path [ ] 2.2 if needed, checkout based on the release date. - [ ] 3. else return null + [x] 3. else return None and raise exception (Cannot get source code) """ + base_dir = Path(self.base_dir) + if not base_dir.exists(): + base_dir.mkdir(parents=True) source_path = None if payload['forge'] == "mvn": if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] if sources_url != "": - source_path = MavenUtils.download_jar(sources_url, self.base_dir) + source_path = MavenUtils.download_jar(sources_url, base_dir) elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: repo_path = payload['repoPath'] repo_type = payload['repoType'] commit_tag = payload['commitTag'] - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, self.base_dir) + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) assert source_path is not None, \ f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." return source_path @@ -83,13 +88,6 @@ def get_source_path(self, payload): assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) return source_path - def clean_up(self): - ''' - TODO - ''' - # if os.path.exists(self.base_dir): - # shutil.rmtree(self.base_dir) - class LizardPackage(Package): diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 0368d8b..5f159ac 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -25,11 +25,8 @@ class MavenUtils: @staticmethod def download_jar(url, base_dir): - base_dir = Path(base_dir) - if not base_dir.exists(): - base_dir.mkdir(parents=True) + tmp_dir = base_dir / "tmp" file_name = base_dir/url.split('/')[-1] - tmp_dir = base_dir/"tmp" r = requests.get(url, allow_redirects=True) open(file_name, 'wb').write(r.content) with ZipFile(file_name, 'r') as zipObj: @@ -38,16 +35,13 @@ def download_jar(url, base_dir): @staticmethod def checkout_version(repo_path, repo_type, version_tag, base_dir): - base_dir = Path(base_dir) - if not base_dir.exists(): - base_dir.mkdir(parents=True) - tmp_dir = base_dir/"tmp" + tmp_dir = base_dir / "tmp" assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) assert repo_path != "", "Empty repo_path." assert version_tag != "", "Empty version_tag." if repo_type == "git": repo = Repo(repo_path) - assert repo.tags[version_tag] is not None + assert repo.tags[version_tag] is not None, "Tag: '{}' does not exist.".format(version_tag) archive_name = version_tag+".zip" archive_file_name = tmp_dir/archive_name repo.git.archive(version_tag, o=archive_file_name) @@ -101,12 +95,32 @@ def tailor_input(payload): cha = { "cha": {} } + depset = { + "depset": [] + } + build_depset = { + "build_depset": [] + } + undeclared_depset = { + "undeclared_depset": [] + } + functions = { + "functions" :{} + } if 'graph' in payload: payload.update(graph) if 'modules' in payload: payload.update(modules) if 'cha' in payload: payload.update(cha) + if 'depset' in payload: + payload.update(depset) + if 'build_depset' in payload: + payload.update(build_depset) + if 'undeclared_depset' in payload: + payload.update(undeclared_depset) + if 'functions' in payload: + payload.update(functions) return payload From 6ee4434ca3ca271768f9df1e06f314a1832140b9 Mon Sep 17 00:00:00 2001 From: cg122 Date: Thu, 29 Oct 2020 14:29:54 +0100 Subject: [PATCH 09/32] Refactored temporary files cleaning. --- rapidplugin/analysis/lizard_analyzer.py | 5 ++-- rapidplugin/tests/test_utils.py | 11 +++++---- rapidplugin/utils/utils.py | 32 +++++++++++++++++-------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index 48c5401..d7de814 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -39,8 +39,8 @@ def analyze(self, payload): forge = payload['forge'] product = payload['groupId'] + ":" + payload['artifactId'] if forge == "mvn" else payload['product'] version = payload['version'] - path = self.get_source_path(payload) - package = LizardPackage(forge, product, version, path) + with self.get_source_path(payload) as path: + package = LizardPackage(forge, product, version, str(path)) metadata = package.metadata() for function in package.functions(): m = {} @@ -86,6 +86,7 @@ def get_source_path(self, payload): assert source_path != "", \ f"Cannot get source code for '{payload['product']}:{payload['version']}', empty 'sourcePath." assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) + source_path = MavenUtils.copy_source(payload['sourcePath'], base_dir) return source_path diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 64af18c..21891cd 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -15,6 +15,8 @@ import os import shutil +from pathlib import Path + import pytest from git import Repo from rapidplugin.utils.utils import MavenUtils, KafkaUtils @@ -41,8 +43,8 @@ def repos(tmp_path_factory): @pytest.mark.parametrize('url', DOWNLOAD_URL_DATA) def test_download_jar(url, sources_dir): - source_path = MavenUtils.download_jar(url, sources_dir) - assert str(source_path) == os.path.join(sources_dir, 'tmp') + with MavenUtils.download_jar(url, sources_dir) as source_path: + assert os.listdir(Path(source_path)) == ['log4j2.xml', 'META-INF', 'ai', 'libai-1.6.12-sources.jar'] @pytest.mark.parametrize('repo_path,repo_type,commit_tag', REPO_PATH_DATA) def test_checkout_version(repo_path, repo_type, commit_tag, sources_dir, repos): @@ -51,8 +53,9 @@ def test_checkout_version(repo_path, repo_type, commit_tag, sources_dir, repos): repo.git.add(".") repo.git.commit(m="first commit.") repo.create_tag('1.0.0') - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) - assert str(source_path) == os.path.join(sources_dir, 'tmp') + with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path: + assert os.listdir(Path(source_path)) == ['1.0.0.zip', 'm1.java'] + PAYLOAD_TAILOR_DATA = [ ({"product": "a"}, {"product": "a"}), diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 5f159ac..c8fd898 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -19,38 +19,50 @@ from svn.local import LocalClient import requests import subprocess as sp +from tempfile import TemporaryDirectory +import os +import shutil class MavenUtils: + @staticmethod + def copy_source(source_path, base_dir): + tmp = TemporaryDirectory(dir=base_dir) + tmp_path = Path(tmp.name) + shutil.copytree(source_path, tmp_path, dirs_exist_ok=True) + return tmp + @staticmethod def download_jar(url, base_dir): - tmp_dir = base_dir / "tmp" - file_name = base_dir/url.split('/')[-1] + tmp = TemporaryDirectory(dir=base_dir) + tmp_path = Path(tmp.name) + file_name = tmp_path/url.split('/')[-1] r = requests.get(url, allow_redirects=True) open(file_name, 'wb').write(r.content) with ZipFile(file_name, 'r') as zipObj: - zipObj.extractall(tmp_dir) - return tmp_dir + zipObj.extractall(tmp_path) + return tmp @staticmethod def checkout_version(repo_path, repo_type, version_tag, base_dir): - tmp_dir = base_dir / "tmp" assert repo_type in {"git", "svn", "hg"}, "Unknown repo type: '{}'.".format(repo_type) assert repo_path != "", "Empty repo_path." assert version_tag != "", "Empty version_tag." + tmp = TemporaryDirectory(dir=base_dir) + tmp_path = Path(tmp.name) if repo_type == "git": repo = Repo(repo_path) assert repo.tags[version_tag] is not None, "Tag: '{}' does not exist.".format(version_tag) archive_name = version_tag+".zip" - archive_file_name = tmp_dir/archive_name + archive_file_name = tmp_path/archive_name repo.git.archive(version_tag, o=archive_file_name) with ZipFile(archive_file_name, 'r') as zipObj: - zipObj.extractall(tmp_dir) + zipObj.extractall(tmp_path) elif repo_type == "svn": return None # r = LocalClient(repo_path) - # r.export(tmp_dir, version_tag) + # r.export(tmp, version_tag) elif repo_type == "hg": return None # cmd = [ @@ -58,11 +70,11 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): # 'archive', # '-r', version_tag, # '-t', 'files', - # tmp_dir + # tmp # ] # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) # o, e = proc.communicate() - return tmp_dir + return tmp class KafkaUtils: From e7bd1f3247e9524f50316c5919a54844f93b8b25 Mon Sep 17 00:00:00 2001 From: cg122 Date: Thu, 29 Oct 2020 15:05:22 +0100 Subject: [PATCH 10/32] Updated dependency of svn. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90d9494..aad3ef4 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ 'fasten', 'lizard', 'requests', - 'gitpython' + 'gitpython', + 'svn' ] ) From 5e8908d0feb87f3084986b57179b51c617cb26fa Mon Sep 17 00:00:00 2001 From: cg122 Date: Thu, 29 Oct 2020 15:14:38 +0100 Subject: [PATCH 11/32] Made testing list sorted. --- rapidplugin/tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 21891cd..5791650 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -44,7 +44,7 @@ def repos(tmp_path_factory): @pytest.mark.parametrize('url', DOWNLOAD_URL_DATA) def test_download_jar(url, sources_dir): with MavenUtils.download_jar(url, sources_dir) as source_path: - assert os.listdir(Path(source_path)) == ['log4j2.xml', 'META-INF', 'ai', 'libai-1.6.12-sources.jar'] + assert sorted(os.listdir(Path(source_path))) == sorted(['log4j2.xml', 'META-INF', 'ai', 'libai-1.6.12-sources.jar']) @pytest.mark.parametrize('repo_path,repo_type,commit_tag', REPO_PATH_DATA) def test_checkout_version(repo_path, repo_type, commit_tag, sources_dir, repos): @@ -54,7 +54,7 @@ def test_checkout_version(repo_path, repo_type, commit_tag, sources_dir, repos): repo.git.commit(m="first commit.") repo.create_tag('1.0.0') with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path: - assert os.listdir(Path(source_path)) == ['1.0.0.zip', 'm1.java'] + assert sorted(os.listdir(Path(source_path))) == sorted(['1.0.0.zip', 'm1.java']) PAYLOAD_TAILOR_DATA = [ From 2f8b678be26aac871f0ba35f5b5d0d6a2ba1e017 Mon Sep 17 00:00:00 2001 From: Magiel Bruntink Date: Fri, 30 Oct 2020 08:29:08 +0100 Subject: [PATCH 12/32] Fix missing docker external in tox config. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index cd822e0..8df73b4 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ passenv = HOME skip_install=true deps = -rrequirements.txt allowlist_externals=python* + docker* docker-compose* flake* pytest* From 8f76811a86c1a5d903ef4c8ea1c239e5910807f6 Mon Sep 17 00:00:00 2001 From: Magiel Bruntink Date: Fri, 30 Oct 2020 08:35:53 +0100 Subject: [PATCH 13/32] Changed paths in integration test cases to fixed paths in the Docker container. --- .../resources/fasten.RepoCloner.out-testdata.txt | 1 - integration_tests/test_plugin.py | 16 +++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) delete mode 100644 integration_tests/resources/fasten.RepoCloner.out-testdata.txt diff --git a/integration_tests/resources/fasten.RepoCloner.out-testdata.txt b/integration_tests/resources/fasten.RepoCloner.out-testdata.txt deleted file mode 100644 index d1642dc..0000000 --- a/integration_tests/resources/fasten.RepoCloner.out-testdata.txt +++ /dev/null @@ -1 +0,0 @@ -{"forge" : "mvn", "groupId": "fasten-project", "artifactId": "fasten", "version": "1.0.0", "repoPath": "~/repos/fasten"} diff --git a/integration_tests/test_plugin.py b/integration_tests/test_plugin.py index 6ba671b..c6a0137 100644 --- a/integration_tests/test_plugin.py +++ b/integration_tests/test_plugin.py @@ -61,10 +61,8 @@ def mock_err(): @pytest.fixture() def plugin_run(mock_in, mock_out, mock_log, mock_err, in_message): - fixed_in_message = fix_sourcePath(in_message, - '/home/plugin/rapidplugin/tests/resources') - mock_in.emit_message(mock_in.produce_topic, fixed_in_message, - "[TEST]", fixed_in_message) + mock_in.emit_message(mock_in.produce_topic, in_message, + "[TEST]", in_message) sleep(2) mock_out.consume_messages() mock_log.consume_messages() @@ -89,7 +87,7 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, # "artifactId": "m1", # "version": "1.0.0", # "sourcesUrl": "", - # "repoPath": "maven/git/m1", + # "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/git/m1", # "repoType": "git", # "commitTag": "1.0.0" # }, @@ -99,7 +97,7 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, # "artifactId": "m2", # "version": "1.0.0", # "sourcesUrl": "", - # "repoPath": "maven/svn/m2", + # "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/svn/m2", # "repoType": "svn", # "commitTag": "1.0.0" # }, @@ -109,7 +107,7 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, # "artifactId": "m3", # "version": "1.0.0", # "sourcesUrl": "", - # "repoPath": "maven/hg/m3", + # "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/hg/m3", # "repoType": "hg", # "commitTag": "1.0.0" # }, @@ -117,13 +115,13 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, "forge": "debian", "product": "d1", "version": "1.0.0", - "sourcePath": "debian/d1" + "sourcePath": "/home/plugin/rapidplugin/tests/resources/debian/d1" }, { "forge": "PyPI", "product": "p1", "version": "1.0.0", - "sourcePath": "pypi/p1" + "sourcePath": "/home/plugin/rapidplugin/tests/resources/pypi/p1" }]) def test_successes(plugin_run, in_message): out, log, err = plugin_run From 00d57f3fdd316504abe701584aa7d1c7df3fa9fb Mon Sep 17 00:00:00 2001 From: Magiel Bruntink Date: Fri, 30 Oct 2020 09:05:43 +0100 Subject: [PATCH 14/32] Moved and split the connect_to_kafka method to superclass. --- rapidplugin/kafka_non_blocking.py | 25 +++++++++++++++++++++++++ rapidplugin/rapid_plugin.py | 15 ++------------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/rapidplugin/kafka_non_blocking.py b/rapidplugin/kafka_non_blocking.py index 0f2f02d..c6c6226 100644 --- a/rapidplugin/kafka_non_blocking.py +++ b/rapidplugin/kafka_non_blocking.py @@ -15,7 +15,9 @@ import json +from time import sleep from kafka import KafkaConsumer +from kafka.errors import NoBrokersAvailable from fasten.plugins.kafka import KafkaPlugin @@ -46,6 +48,29 @@ def set_consumer(self): consumer_timeout_ms=self.consumer_timeout_ms ) + def set_consumer_with_retry(self): + while True: + try: + self.set_consumer() + self.set_producer() + except NoBrokersAvailable: + self.err("Could not connect consumer to Kafka, re-trying...") + else: + self.log("Connected consumer to Kafka successfully.") + break + sleep(self.consumption_delay_sec) + + def set_producer_with_retry(self): + while True: + try: + self.set_producer() + except NoBrokersAvailable: + self.err("Could not connect producer to Kafka, re-trying...") + else: + self.log("Connected producer to Kafka successfully.") + break + sleep(self.consumption_delay_sec) + def skip_messages(self): assert self.consumer is not None, "Consumer needs to be set before messages can ben consumed." for m in self.consumer: diff --git a/rapidplugin/rapid_plugin.py b/rapidplugin/rapid_plugin.py index f94641c..115ad33 100644 --- a/rapidplugin/rapid_plugin.py +++ b/rapidplugin/rapid_plugin.py @@ -40,19 +40,8 @@ def __init__(self, name, version, description, plugin_config): self.sources_dir = self.plugin_config.get_config_value('sources_dir') self.consumer_timeout_ms = self.plugin_config.get_config_value('consumer_timeout_ms') self.consumption_delay_sec = self.plugin_config.get_config_value('consumption_delay_sec') - self.connect_to_kafka() - - def connect_to_kafka(self): - while True: - try: - self.set_consumer() - self.set_producer() - except Exception as e: - self.err("Could not connect to Kafka, re-trying...") - else: - self.log("Connected to Kafka successfully.") - break - sleep(self.consumption_delay_sec) + self.set_consumer_with_retry() + self.set_producer_with_retry() def name(self): return self._name From 8a8056cd38a81c39c55f4373f981aeaccac03bf8 Mon Sep 17 00:00:00 2001 From: cg122 Date: Fri, 30 Oct 2020 09:57:33 +0100 Subject: [PATCH 15/32] Refactored utils. --- rapidplugin/analysis/lizard_analyzer.py | 40 +------ rapidplugin/utils/utils.py | 143 +++++++++++++++--------- 2 files changed, 90 insertions(+), 93 deletions(-) diff --git a/rapidplugin/analysis/lizard_analyzer.py b/rapidplugin/analysis/lizard_analyzer.py index d7de814..fec54b6 100644 --- a/rapidplugin/analysis/lizard_analyzer.py +++ b/rapidplugin/analysis/lizard_analyzer.py @@ -39,7 +39,7 @@ def analyze(self, payload): forge = payload['forge'] product = payload['groupId'] + ":" + payload['artifactId'] if forge == "mvn" else payload['product'] version = payload['version'] - with self.get_source_path(payload) as path: + with MavenUtils.get_source_path(payload, self.base_dir) as path: package = LizardPackage(forge, product, version, str(path)) metadata = package.metadata() for function in package.functions(): @@ -51,44 +51,6 @@ def analyze(self, payload): logger.debug("callable: {}".format(m) + '\n') return out_payloads - def get_source_path(self, payload): - """ - TODO: consider moving this to a utility class. - For maven, the order to get source code path from different sources: - [x] 1. if *-sources.jar is valid, download, - uncompress and return the path to the source code - [x] 2. else if repoPath is not empty, and - [x] 2.1 if commit tag is valid, checkout based on tag and return the path - [ ] 2.2 if needed, checkout based on the release date. - [x] 3. else return None and raise exception (Cannot get source code) - """ - base_dir = Path(self.base_dir) - if not base_dir.exists(): - base_dir.mkdir(parents=True) - source_path = None - if payload['forge'] == "mvn": - if 'sourcesUrl' in payload: - sources_url = payload['sourcesUrl'] - if sources_url != "": - source_path = MavenUtils.download_jar(sources_url, base_dir) - elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: - repo_path = payload['repoPath'] - repo_type = payload['repoType'] - commit_tag = payload['commitTag'] - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) - assert source_path is not None, \ - f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." - return source_path - else: - assert 'sourcePath' in payload, \ - f"Cannot get source code for '{payload['product']}:{payload['version']}', missing 'sourcePath'." - source_path = payload['sourcePath'] - assert source_path != "", \ - f"Cannot get source code for '{payload['product']}:{payload['version']}', empty 'sourcePath." - assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) - source_path = MavenUtils.copy_source(payload['sourcePath'], base_dir) - return source_path - class LizardPackage(Package): diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index c8fd898..6452fe6 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -26,6 +26,53 @@ class MavenUtils: + @staticmethod + def get_source_path(payload, base_dir): + """ + TODO: consider moving this to a utility class. + For maven, the order to get source code path from different sources: + [x] 1. if *-sources.jar is valid, download, + uncompress and return the path to the source code + [x] 2. else if repoPath is not empty, and + [x] 2.1 if commit tag is valid, checkout based on tag and return the path + [ ] 2.2 if needed, checkout based on the release date. + [x] 3. else return None and raise exception (Cannot get source code) + """ + base_dir = Path(base_dir) + if not base_dir.exists(): + base_dir.mkdir(parents=True) + if payload['forge'] == "mvn": + source_path = MavenUtils.get_source_mvn(payload, base_dir) + else: + source_path = MavenUtils.get_source_other(payload, base_dir) + return source_path + + @staticmethod + def get_source_mvn(payload, base_dir): + if 'sourcesUrl' in payload: + sources_url = payload['sourcesUrl'] + if sources_url != "": + source_path = MavenUtils.download_jar(sources_url, base_dir) + elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: + repo_path = payload['repoPath'] + repo_type = payload['repoType'] + commit_tag = payload['commitTag'] + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) + assert source_path is not None, \ + f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." + return source_path + + @staticmethod + def get_source_other(payload, base_dir): + assert 'sourcePath' in payload, \ + f"Cannot get source code for '{payload['product']}:{payload['version']}', missing 'sourcePath'." + source_path = payload['sourcePath'] + assert source_path != "", \ + f"Cannot get source code for '{payload['product']}:{payload['version']}', empty 'sourcePath." + assert os.path.isabs(source_path), "sourcePath: '{}' is not an absolute path!".format(source_path) + source_path = MavenUtils.copy_source(payload['sourcePath'], base_dir) + return source_path + @staticmethod def copy_source(source_path, base_dir): tmp = TemporaryDirectory(dir=base_dir) @@ -52,30 +99,41 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): tmp = TemporaryDirectory(dir=base_dir) tmp_path = Path(tmp.name) if repo_type == "git": - repo = Repo(repo_path) - assert repo.tags[version_tag] is not None, "Tag: '{}' does not exist.".format(version_tag) - archive_name = version_tag+".zip" - archive_file_name = tmp_path/archive_name - repo.git.archive(version_tag, o=archive_file_name) - with ZipFile(archive_file_name, 'r') as zipObj: - zipObj.extractall(tmp_path) + MavenUtils.git_checkout(repo_path, version_tag, tmp_path) elif repo_type == "svn": - return None - # r = LocalClient(repo_path) - # r.export(tmp, version_tag) + MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) elif repo_type == "hg": - return None - # cmd = [ - # 'hg', - # 'archive', - # '-r', version_tag, - # '-t', 'files', - # tmp - # ] - # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - # o, e = proc.communicate() + MavenUtils.hg_checkout() return tmp + @staticmethod + def git_checkout(repo_path, version_tag, tmp_path): + repo = Repo(repo_path) + assert repo.tags[version_tag] is not None, "Tag: '{}' does not exist.".format(version_tag) + archive_name = version_tag+".zip" + archive_file_name = tmp_path/archive_name + repo.git.archive(version_tag, o=archive_file_name) + with ZipFile(archive_file_name, 'r') as zipObj: + zipObj.extractall(tmp_path) + + @staticmethod + def svn_checkout(repo_path, version_tag, tmp_path): + return None + # r = LocalClient(repo_path) + # r.export(tmp, version_tag) + + @staticmethod + def hg_checkout(repo_path, version_tag, tmp_path): + return None + # cmd = [ + # 'hg', + # 'archive', + # '-r', version_tag, + # '-t', 'files', + # tmp + # ] + # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + # o, e = proc.communicate() class KafkaUtils: @staticmethod @@ -98,41 +156,18 @@ def tailor_input(payload): to avoid (big) call graph data adding to 'input' field in the produced topics. """ - graph = { - "graph": {} - } - modules = { - "modules": {} - } - cha = { - "cha": {} - } - depset = { - "depset": [] - } - build_depset = { - "build_depset": [] - } - undeclared_depset = { - "undeclared_depset": [] - } - functions = { - "functions" :{} + tailor = { + "graph": {}, + "modules": {}, + "cha": {}, + "depset": [], + "build_depset": [], + "undeclared_depset": [], + "functions": {} } - if 'graph' in payload: - payload.update(graph) - if 'modules' in payload: - payload.update(modules) - if 'cha' in payload: - payload.update(cha) - if 'depset' in payload: - payload.update(depset) - if 'build_depset' in payload: - payload.update(build_depset) - if 'undeclared_depset' in payload: - payload.update(undeclared_depset) - if 'functions' in payload: - payload.update(functions) + for key in tailor.keys(): + if key in payload: + payload[key] = tailor[key] return payload From 0471c8a4e0ff1f4eb81b2f04f89f017d83364b1f Mon Sep 17 00:00:00 2001 From: cg122 Date: Fri, 30 Oct 2020 10:12:59 +0100 Subject: [PATCH 16/32] Fixed missing parameters. --- rapidplugin/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 6452fe6..4141070 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -103,7 +103,7 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): elif repo_type == "svn": MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) elif repo_type == "hg": - MavenUtils.hg_checkout() + MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) return tmp @staticmethod From bbad18f66b2aee50ce06dbf5be70c667488c107c Mon Sep 17 00:00:00 2001 From: cg122 Date: Fri, 30 Oct 2020 10:17:19 +0100 Subject: [PATCH 17/32] Fixed return value of checkout. --- rapidplugin/utils/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 4141070..b454461 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -101,9 +101,9 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): if repo_type == "git": MavenUtils.git_checkout(repo_path, version_tag, tmp_path) elif repo_type == "svn": - MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) + return MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) elif repo_type == "hg": - MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) + return MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) return tmp @staticmethod @@ -120,7 +120,7 @@ def git_checkout(repo_path, version_tag, tmp_path): def svn_checkout(repo_path, version_tag, tmp_path): return None # r = LocalClient(repo_path) - # r.export(tmp, version_tag) + # r.export(tmp_path, version_tag) @staticmethod def hg_checkout(repo_path, version_tag, tmp_path): @@ -130,7 +130,7 @@ def hg_checkout(repo_path, version_tag, tmp_path): # 'archive', # '-r', version_tag, # '-t', 'files', - # tmp + # tmp_path # ] # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) # o, e = proc.communicate() From 01ef03ba6b1bb1cd0d51098b348469353363d36f Mon Sep 17 00:00:00 2001 From: cg122 Date: Fri, 30 Oct 2020 11:55:47 +0100 Subject: [PATCH 18/32] Added exceptions for unsupported repo(svn, hg) --- rapidplugin/utils/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index b454461..e149d9d 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -29,7 +29,6 @@ class MavenUtils: @staticmethod def get_source_path(payload, base_dir): """ - TODO: consider moving this to a utility class. For maven, the order to get source code path from different sources: [x] 1. if *-sources.jar is valid, download, uncompress and return the path to the source code @@ -49,6 +48,7 @@ def get_source_path(payload, base_dir): @staticmethod def get_source_mvn(payload, base_dir): + source_path = None if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] if sources_url != "": @@ -101,9 +101,9 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): if repo_type == "git": MavenUtils.git_checkout(repo_path, version_tag, tmp_path) elif repo_type == "svn": - return MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) + MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) elif repo_type == "hg": - return MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) + MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) return tmp @staticmethod @@ -118,13 +118,15 @@ def git_checkout(repo_path, version_tag, tmp_path): @staticmethod def svn_checkout(repo_path, version_tag, tmp_path): - return None + repo_type = "svn" + raise Exception('Unsupported repo_type:{}.'.format(repo_type)) # r = LocalClient(repo_path) # r.export(tmp_path, version_tag) @staticmethod def hg_checkout(repo_path, version_tag, tmp_path): - return None + repo_type = "hg" + raise Exception('Unsupported repo_type:{}.'.format(repo_type)) # cmd = [ # 'hg', # 'archive', @@ -135,6 +137,7 @@ def hg_checkout(repo_path, version_tag, tmp_path): # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) # o, e = proc.communicate() + class KafkaUtils: @staticmethod def validate_message(payload): From 316d9e713cdb2d1dc28a4ab78404f51fa6084cae Mon Sep 17 00:00:00 2001 From: cg122 Date: Fri, 30 Oct 2020 12:05:15 +0100 Subject: [PATCH 19/32] Added exceptions for unsupported repo(svn, hg) and testing. --- rapidplugin/tests/test_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 5791650..0351324 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -47,7 +47,7 @@ def test_download_jar(url, sources_dir): assert sorted(os.listdir(Path(source_path))) == sorted(['log4j2.xml', 'META-INF', 'ai', 'libai-1.6.12-sources.jar']) @pytest.mark.parametrize('repo_path,repo_type,commit_tag', REPO_PATH_DATA) -def test_checkout_version(repo_path, repo_type, commit_tag, sources_dir, repos): +def test_checkout_success(repo_path, repo_type, commit_tag, sources_dir, repos): repo_path = os.path.join(repos, repo_path) repo = Repo.init(repo_path) repo.git.add(".") @@ -56,6 +56,12 @@ def test_checkout_version(repo_path, repo_type, commit_tag, sources_dir, repos): with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path: assert sorted(os.listdir(Path(source_path))) == sorted(['1.0.0.zip', 'm1.java']) +@pytest.mark.parametrize('repo_path,repo_type,commit_tag', + [("maven/svn/m2", "svn", "1.0.0"),("maven/hg/m3", "hg", "1.0.0")]) +def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir): + with pytest.raises(Exception): + MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) + PAYLOAD_TAILOR_DATA = [ ({"product": "a"}, {"product": "a"}), From d4ac7e2030111966cf204744d25db2a4d06b4a6e Mon Sep 17 00:00:00 2001 From: cg122 Date: Fri, 30 Oct 2020 12:14:57 +0100 Subject: [PATCH 20/32] Added more testing for failure cases. --- rapidplugin/tests/test_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 0351324..5c109ab 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -42,10 +42,15 @@ def repos(tmp_path_factory): yield tmp @pytest.mark.parametrize('url', DOWNLOAD_URL_DATA) -def test_download_jar(url, sources_dir): +def test_download_success(url, sources_dir): with MavenUtils.download_jar(url, sources_dir) as source_path: assert sorted(os.listdir(Path(source_path))) == sorted(['log4j2.xml', 'META-INF', 'ai', 'libai-1.6.12-sources.jar']) +@pytest.mark.parametrize('url', ["http://abc.com/def.jar"]) +def test_download_fail(url, sources_dir): + with pytest.raises(Exception): + MavenUtils.download_jar(url, sources_dir) + @pytest.mark.parametrize('repo_path,repo_type,commit_tag', REPO_PATH_DATA) def test_checkout_success(repo_path, repo_type, commit_tag, sources_dir, repos): repo_path = os.path.join(repos, repo_path) From ff7688950e731b8395527e84f565da822323f395 Mon Sep 17 00:00:00 2001 From: Magiel Bruntink Date: Fri, 30 Oct 2020 13:21:58 +0100 Subject: [PATCH 21/32] Fixed tox not knowing all env variables. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8df73b4..0d9a494 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = py38 [testenv] -passenv = HOME +passenv = * skip_install=true deps = -rrequirements.txt allowlist_externals=python* From eacb3b8a594afb48f37e0bb786fd03a7277779ab Mon Sep 17 00:00:00 2001 From: cg122 Date: Mon, 2 Nov 2020 19:38:04 +0100 Subject: [PATCH 22/32] Disabled svn feature and added more testing --- integration_tests/test_plugin.py | 30 +++++++++++++++++++ rapidplugin/tests/test_utils.py | 10 +++++-- rapidplugin/utils/utils.py | 50 ++++++++++++++++++-------------- 3 files changed, 66 insertions(+), 24 deletions(-) diff --git a/integration_tests/test_plugin.py b/integration_tests/test_plugin.py index 6ba671b..5b5abd8 100644 --- a/integration_tests/test_plugin.py +++ b/integration_tests/test_plugin.py @@ -124,6 +124,16 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, "product": "p1", "version": "1.0.0", "sourcePath": "pypi/p1" + }, + { + "forge": "mvn", + "groupId": "test-mvn", + "artifactId": "m3", + "version": "1.0.0", + "sourcesUrl": "", + "repoPath": "maven/hg/m3", + "repoType": "hg", + "commitTag": "1.0.0" }]) def test_successes(plugin_run, in_message): out, log, err = plugin_run @@ -145,6 +155,26 @@ def test_successes(plugin_run, in_message): "forge": "PyPI", "product": "p1", "version": "1.0.0" + }, + { + "forge": "mvn", + "groupId": "test-mvn", + "artifactId": "m2", + "version": "1.0.0", + "sourcesUrl": "", + "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/svn/m2", + "repoType": "svn", + "commitTag": "1.0.0" + }, + { + "forge": "mvn", + "groupId": "test-mvn", + "artifactId": "m3", + "version": "1.0.1", + "sourcesUrl": "", + "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/hg/m3", + "repoType": "hg", + "commitTag": "1.0.1" }]) def test_failures(plugin_run, in_message): out, log, err = plugin_run diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 5c109ab..38d6ddf 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -52,7 +52,7 @@ def test_download_fail(url, sources_dir): MavenUtils.download_jar(url, sources_dir) @pytest.mark.parametrize('repo_path,repo_type,commit_tag', REPO_PATH_DATA) -def test_checkout_success(repo_path, repo_type, commit_tag, sources_dir, repos): +def test_checkout_git(repo_path, repo_type, commit_tag, sources_dir, repos): repo_path = os.path.join(repos, repo_path) repo = Repo.init(repo_path) repo.git.add(".") @@ -61,8 +61,14 @@ def test_checkout_success(repo_path, repo_type, commit_tag, sources_dir, repos): with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path: assert sorted(os.listdir(Path(source_path))) == sorted(['1.0.0.zip', 'm1.java']) +@pytest.mark.parametrize('repo_path,repo_type,commit_tag', [("maven/hg/m3", "hg", "1.0.0")]) +def test_checkout_hg(repo_path, repo_type, commit_tag, sources_dir, repos): + repo_path = os.path.join(repos, repo_path) + with MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) as source_path: + assert sorted(os.listdir(source_path)) == sorted(['m3.java', '.hg_archival.txt']) + @pytest.mark.parametrize('repo_path,repo_type,commit_tag', - [("maven/svn/m2", "svn", "1.0.0"),("maven/hg/m3", "hg", "1.0.0")]) + [("maven/git/m1", "git", "1.0.1"),("maven/hg/m3", "hg", "1.0.1")]) def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir): with pytest.raises(Exception): MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index e149d9d..82900d0 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -44,6 +44,8 @@ def get_source_path(payload, base_dir): source_path = MavenUtils.get_source_mvn(payload, base_dir) else: source_path = MavenUtils.get_source_other(payload, base_dir) + assert source_path is not None, \ + f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." return source_path @staticmethod @@ -58,8 +60,8 @@ def get_source_mvn(payload, base_dir): repo_type = payload['repoType'] commit_tag = payload['commitTag'] source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) - assert source_path is not None, \ - f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." + # assert source_path is not None, \ + # f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." return source_path @staticmethod @@ -98,12 +100,15 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): assert version_tag != "", "Empty version_tag." tmp = TemporaryDirectory(dir=base_dir) tmp_path = Path(tmp.name) - if repo_type == "git": - MavenUtils.git_checkout(repo_path, version_tag, tmp_path) - elif repo_type == "svn": - MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) - elif repo_type == "hg": - MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) + try: + if repo_type == "git": + MavenUtils.git_checkout(repo_path, version_tag, tmp_path) + elif repo_type == "svn": + MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) + elif repo_type == "hg": + MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) + except Exception as e: + raise e return tmp @staticmethod @@ -118,25 +123,26 @@ def git_checkout(repo_path, version_tag, tmp_path): @staticmethod def svn_checkout(repo_path, version_tag, tmp_path): - repo_type = "svn" - raise Exception('Unsupported repo_type:{}.'.format(repo_type)) + raise NotImplemented # r = LocalClient(repo_path) # r.export(tmp_path, version_tag) @staticmethod def hg_checkout(repo_path, version_tag, tmp_path): - repo_type = "hg" - raise Exception('Unsupported repo_type:{}.'.format(repo_type)) - # cmd = [ - # 'hg', - # 'archive', - # '-r', version_tag, - # '-t', 'files', - # tmp_path - # ] - # proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - # o, e = proc.communicate() - + os.chdir(repo_path) + print(os.listdir(repo_path)) + cmd = [ + 'hg', + 'archive', + '-r', version_tag, + '-t', 'files', + tmp_path + ] + try: + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + out, err = proc.communicate() + except Exception as e: + raise e("Check out error") class KafkaUtils: @staticmethod From a6bc978dd00249bb9306386785280ebd516c0cbc Mon Sep 17 00:00:00 2001 From: cg122 Date: Mon, 2 Nov 2020 21:46:47 +0100 Subject: [PATCH 23/32] Disabled svn feature and added more testing --- integration_tests/test_plugin.py | 14 +----- .../tests/resources/maven/hg/m3/.hgtags | 1 + .../tests/resources/maven/hg/m3/m3.java | 5 +++ rapidplugin/tests/sources.py | 5 +++ rapidplugin/tests/test_analyzer.py | 4 +- rapidplugin/utils/utils.py | 45 +++++++------------ 6 files changed, 33 insertions(+), 41 deletions(-) create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hgtags create mode 100644 rapidplugin/tests/resources/maven/hg/m3/m3.java diff --git a/integration_tests/test_plugin.py b/integration_tests/test_plugin.py index 5b5abd8..b0cb422 100644 --- a/integration_tests/test_plugin.py +++ b/integration_tests/test_plugin.py @@ -124,16 +124,6 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, "product": "p1", "version": "1.0.0", "sourcePath": "pypi/p1" - }, - { - "forge": "mvn", - "groupId": "test-mvn", - "artifactId": "m3", - "version": "1.0.0", - "sourcesUrl": "", - "repoPath": "maven/hg/m3", - "repoType": "hg", - "commitTag": "1.0.0" }]) def test_successes(plugin_run, in_message): out, log, err = plugin_run @@ -162,7 +152,7 @@ def test_successes(plugin_run, in_message): "artifactId": "m2", "version": "1.0.0", "sourcesUrl": "", - "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/svn/m2", + "repoPath": "maven/svn/m2", "repoType": "svn", "commitTag": "1.0.0" }, @@ -172,7 +162,7 @@ def test_successes(plugin_run, in_message): "artifactId": "m3", "version": "1.0.1", "sourcesUrl": "", - "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/hg/m3", + "repoPath": "maven/hg/m3", "repoType": "hg", "commitTag": "1.0.1" }]) diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hgtags b/rapidplugin/tests/resources/maven/hg/m3/.hgtags new file mode 100644 index 0000000..4aae4bd --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/.hgtags @@ -0,0 +1 @@ +6cb73187f2c9c3983baac1e38162bd1ea64d608f 1.0.0 diff --git a/rapidplugin/tests/resources/maven/hg/m3/m3.java b/rapidplugin/tests/resources/maven/hg/m3/m3.java new file mode 100644 index 0000000..dd9e484 --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/m3.java @@ -0,0 +1,5 @@ +class M3 { + public static void main(String[] args) { + System.out.println("M3 from hg!"); + } +} \ No newline at end of file diff --git a/rapidplugin/tests/sources.py b/rapidplugin/tests/sources.py index 5136c72..ef7896d 100644 --- a/rapidplugin/tests/sources.py +++ b/rapidplugin/tests/sources.py @@ -14,4 +14,9 @@ def fix_sourcePath(record, tmp_sources_path): if "sourcePath" in record: sourcePath = record["sourcePath"] record.update({"sourcePath": os.path.join(tmp_sources_path, sourcePath)}) + if "repoPath" in record: + repoPath = record["repoPath"] + record.update({"repoPath": os.path.join(tmp_sources_path, repoPath)}) return record + + diff --git a/rapidplugin/tests/test_analyzer.py b/rapidplugin/tests/test_analyzer.py index b2de166..8da022c 100644 --- a/rapidplugin/tests/test_analyzer.py +++ b/rapidplugin/tests/test_analyzer.py @@ -81,13 +81,15 @@ def analyzer(sources): # List of (payload, function_count) pairs FUNCTION_COUNT_DATA = [ (mvn_libai_1_6_12, 411), + # (mvn_message_with_git_repo, 1), + (mvn_message_with_hg_repo, 1), (debian_message, 1), (pypi_message, 1) ] # List of (payload, start_line, end_line) tuples FUNCTION_LINE_DATA = [ - # (mvn_message_with_repo, 2, 4), + (mvn_message_with_hg_repo, 2, 4), (debian_message, 3, 3), (pypi_message, 1, 2) ] diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 82900d0..c926834 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -44,25 +44,20 @@ def get_source_path(payload, base_dir): source_path = MavenUtils.get_source_mvn(payload, base_dir) else: source_path = MavenUtils.get_source_other(payload, base_dir) - assert source_path is not None, \ - f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." return source_path @staticmethod def get_source_mvn(payload, base_dir): - source_path = None if 'sourcesUrl' in payload: sources_url = payload['sourcesUrl'] if sources_url != "": - source_path = MavenUtils.download_jar(sources_url, base_dir) - elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: - repo_path = payload['repoPath'] - repo_type = payload['repoType'] - commit_tag = payload['commitTag'] - source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) - # assert source_path is not None, \ - # f"Cannot get source code for '{payload['groupId']}:{payload['artifactId']}:{payload['version']}'." - return source_path + return MavenUtils.download_jar(sources_url, base_dir) + elif 'repoPath' in payload and 'commitTag' in payload and 'repoType' in payload: + repo_path = payload['repoPath'] + repo_type = payload['repoType'] + commit_tag = payload['commitTag'] + source_path = MavenUtils.checkout_version(repo_path, repo_type, commit_tag, base_dir) + return source_path @staticmethod def get_source_other(payload, base_dir): @@ -100,15 +95,12 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): assert version_tag != "", "Empty version_tag." tmp = TemporaryDirectory(dir=base_dir) tmp_path = Path(tmp.name) - try: - if repo_type == "git": - MavenUtils.git_checkout(repo_path, version_tag, tmp_path) - elif repo_type == "svn": - MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) - elif repo_type == "hg": - MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) - except Exception as e: - raise e + if repo_type == "git": + MavenUtils.git_checkout(repo_path, version_tag, tmp_path) + elif repo_type == "svn": + MavenUtils.svn_checkout(repo_path, version_tag, tmp_path) + elif repo_type == "hg": + MavenUtils.hg_checkout(repo_path, version_tag, tmp_path) return tmp @staticmethod @@ -123,14 +115,14 @@ def git_checkout(repo_path, version_tag, tmp_path): @staticmethod def svn_checkout(repo_path, version_tag, tmp_path): - raise NotImplemented + raise NotImplementedError + # 'svn export' does not support tag # r = LocalClient(repo_path) # r.export(tmp_path, version_tag) @staticmethod def hg_checkout(repo_path, version_tag, tmp_path): os.chdir(repo_path) - print(os.listdir(repo_path)) cmd = [ 'hg', 'archive', @@ -138,11 +130,8 @@ def hg_checkout(repo_path, version_tag, tmp_path): '-t', 'files', tmp_path ] - try: - proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - out, err = proc.communicate() - except Exception as e: - raise e("Check out error") + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + out, err = proc.communicate() class KafkaUtils: @staticmethod From 9e4286e57fb6a76115ae2914fcd837f7d749db11 Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 3 Nov 2020 00:08:42 +0100 Subject: [PATCH 24/32] Fixed working directory issue in hg checkout --- rapidplugin/tests/sources.py | 5 ----- rapidplugin/tests/test_analyzer.py | 6 +++--- rapidplugin/tests/test_utils.py | 10 ++++++---- rapidplugin/utils/utils.py | 2 ++ 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/rapidplugin/tests/sources.py b/rapidplugin/tests/sources.py index ef7896d..5136c72 100644 --- a/rapidplugin/tests/sources.py +++ b/rapidplugin/tests/sources.py @@ -14,9 +14,4 @@ def fix_sourcePath(record, tmp_sources_path): if "sourcePath" in record: sourcePath = record["sourcePath"] record.update({"sourcePath": os.path.join(tmp_sources_path, sourcePath)}) - if "repoPath" in record: - repoPath = record["repoPath"] - record.update({"repoPath": os.path.join(tmp_sources_path, repoPath)}) return record - - diff --git a/rapidplugin/tests/test_analyzer.py b/rapidplugin/tests/test_analyzer.py index 8da022c..8732a65 100644 --- a/rapidplugin/tests/test_analyzer.py +++ b/rapidplugin/tests/test_analyzer.py @@ -61,7 +61,7 @@ def analyzer(sources): "artifactId": "m3", "version": "1.0.0", "sourcesUrl": "", - "repoPath": "maven/hg/m3", + "repoPath": "rapidplugin/tests/resources/maven/hg/m3", "repoType": "hg", "commitTag": "1.0.0" } @@ -89,14 +89,14 @@ def analyzer(sources): # List of (payload, start_line, end_line) tuples FUNCTION_LINE_DATA = [ - (mvn_message_with_hg_repo, 2, 4), + # (mvn_message_with_hg_repo, 2, 4), (debian_message, 3, 3), (pypi_message, 1, 2) ] # List of (payload, nloc, complexity, token_count) tuples FUNCTION_METRICS_DATA = [ - # (mvn_message_with_repo, 3, 1, 18), + # (mvn_message_with_hg_repo, 3, 1, 18), (debian_message, 1, 1, 5), (pypi_message, 2, 1, 5) ] diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 38d6ddf..a67fe09 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -21,6 +21,7 @@ from git import Repo from rapidplugin.utils.utils import MavenUtils, KafkaUtils + DOWNLOAD_URL_DATA = [ ("https://repo1.maven.org/maven2/ai/api/libai/1.6.12/libai-1.6.12-sources.jar") ] @@ -31,16 +32,17 @@ # ("maven/hg/m3", "hg", "1.0.0") ] -@pytest.fixture(scope='session') -def sources_dir(tmp_path_factory): - yield tmp_path_factory.mktemp("sources") - @pytest.fixture(scope='session') def repos(tmp_path_factory): tmp = tmp_path_factory.mktemp("repos") + print(f"cwd: {os.getcwd()}") shutil.copytree('rapidplugin/tests/resources', tmp, dirs_exist_ok=True) yield tmp +@pytest.fixture(scope='session') +def sources_dir(tmp_path_factory): + yield tmp_path_factory.mktemp("sources") + @pytest.mark.parametrize('url', DOWNLOAD_URL_DATA) def test_download_success(url, sources_dir): with MavenUtils.download_jar(url, sources_dir) as source_path: diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index c926834..f8a40e1 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -122,6 +122,7 @@ def svn_checkout(repo_path, version_tag, tmp_path): @staticmethod def hg_checkout(repo_path, version_tag, tmp_path): + wd = os.getcwd() os.chdir(repo_path) cmd = [ 'hg', @@ -132,6 +133,7 @@ def hg_checkout(repo_path, version_tag, tmp_path): ] proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) out, err = proc.communicate() + os.chdir(wd) class KafkaUtils: @staticmethod From 38e52b0d0f17e343793a4c4b2e0e2b61e4ff892d Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 3 Nov 2020 07:55:33 +0100 Subject: [PATCH 25/32] Added more unit testing. --- rapidplugin/tests/test_analyzer.py | 4 ++-- rapidplugin/tests/test_utils.py | 7 ++++--- rapidplugin/utils/utils.py | 4 +++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/rapidplugin/tests/test_analyzer.py b/rapidplugin/tests/test_analyzer.py index 8732a65..2028bd3 100644 --- a/rapidplugin/tests/test_analyzer.py +++ b/rapidplugin/tests/test_analyzer.py @@ -89,14 +89,14 @@ def analyzer(sources): # List of (payload, start_line, end_line) tuples FUNCTION_LINE_DATA = [ - # (mvn_message_with_hg_repo, 2, 4), + (mvn_message_with_hg_repo, 2, 4), (debian_message, 3, 3), (pypi_message, 1, 2) ] # List of (payload, nloc, complexity, token_count) tuples FUNCTION_METRICS_DATA = [ - # (mvn_message_with_hg_repo, 3, 1, 18), + (mvn_message_with_hg_repo, 3, 1, 18), (debian_message, 1, 1, 5), (pypi_message, 2, 1, 5) ] diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index a67fe09..4093dd6 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -35,7 +35,6 @@ @pytest.fixture(scope='session') def repos(tmp_path_factory): tmp = tmp_path_factory.mktemp("repos") - print(f"cwd: {os.getcwd()}") shutil.copytree('rapidplugin/tests/resources', tmp, dirs_exist_ok=True) yield tmp @@ -70,9 +69,11 @@ def test_checkout_hg(repo_path, repo_type, commit_tag, sources_dir, repos): assert sorted(os.listdir(source_path)) == sorted(['m3.java', '.hg_archival.txt']) @pytest.mark.parametrize('repo_path,repo_type,commit_tag', - [("maven/git/m1", "git", "1.0.1"),("maven/hg/m3", "hg", "1.0.1")]) + [("maven/git/m1", "git", "1.0.1"), + ("rapidplugin/tests/resources/maven/hg/m3", "hg", "1.0.1"), + ("maven/svn/m2", "svn", "1.0.0")]) def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir): - with pytest.raises(Exception): + with pytest.raises(Exception) as e: MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index f8a40e1..0c2309e 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -115,7 +115,7 @@ def git_checkout(repo_path, version_tag, tmp_path): @staticmethod def svn_checkout(repo_path, version_tag, tmp_path): - raise NotImplementedError + raise NotImplementedError("Svn repo not supported.") # 'svn export' does not support tag # r = LocalClient(repo_path) # r.export(tmp_path, version_tag) @@ -134,6 +134,8 @@ def hg_checkout(repo_path, version_tag, tmp_path): proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) out, err = proc.communicate() os.chdir(wd) + if proc.returncode !=0: + raise Exception(str(err)) class KafkaUtils: @staticmethod From 91368f3ce3e934bf1f703cb6cb6880dde47a9780 Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 3 Nov 2020 08:09:37 +0100 Subject: [PATCH 26/32] Added hg metadata --- .../tests/resources/maven/hg/m3/.hg/00changelog.i | Bin 0 -> 57 bytes .../maven/hg/m3/.hg/cache/branch2-served | 2 ++ .../resources/maven/hg/m3/.hg/cache/hgtagsfnodes1 | 2 ++ .../resources/maven/hg/m3/.hg/cache/rbc-names-v1 | 1 + .../resources/maven/hg/m3/.hg/cache/rbc-revs-v1 | Bin 0 -> 16 bytes .../resources/maven/hg/m3/.hg/cache/tags2-visible | 2 ++ .../tests/resources/maven/hg/m3/.hg/dirstate | Bin 0 -> 88 bytes .../resources/maven/hg/m3/.hg/last-message.txt | 1 + .../tests/resources/maven/hg/m3/.hg/requires | 6 ++++++ .../resources/maven/hg/m3/.hg/store/00changelog.i | Bin 0 -> 357 bytes .../resources/maven/hg/m3/.hg/store/00manifest.i | Bin 0 -> 238 bytes .../maven/hg/m3/.hg/store/data/m3.java.i | Bin 0 -> 162 bytes .../maven/hg/m3/.hg/store/data/~2ehgtags.i | Bin 0 -> 112 bytes .../tests/resources/maven/hg/m3/.hg/store/fncache | 2 ++ .../resources/maven/hg/m3/.hg/store/phaseroots | 1 + .../tests/resources/maven/hg/m3/.hg/store/undo | Bin 0 -> 52 bytes .../maven/hg/m3/.hg/store/undo.backup.fncache | 1 + .../maven/hg/m3/.hg/store/undo.backupfiles | Bin 0 -> 71 bytes .../maven/hg/m3/.hg/store/undo.phaseroots | 1 + .../maven/hg/m3/.hg/undo.backup.dirstate | Bin 0 -> 88 bytes .../resources/maven/hg/m3/.hg/undo.bookmarks | 0 .../tests/resources/maven/hg/m3/.hg/undo.branch | 1 + .../tests/resources/maven/hg/m3/.hg/undo.desc | 2 ++ .../tests/resources/maven/hg/m3/.hg/undo.dirstate | Bin 0 -> 88 bytes .../resources/maven/hg/m3/.hg/wcache/checkisexec | 0 .../resources/maven/hg/m3/.hg/wcache/checklink | 1 + .../maven/hg/m3/.hg/wcache/checklink-target | 0 .../resources/maven/hg/m3/.hg/wcache/checknoexec | 0 .../maven/hg/m3/.hg/wcache/manifestfulltextcache | Bin 0 -> 195 bytes 29 files changed, 23 insertions(+) create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/00changelog.i create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/cache/branch2-served create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/cache/hgtagsfnodes1 create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/cache/rbc-names-v1 create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/cache/rbc-revs-v1 create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/cache/tags2-visible create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/dirstate create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/last-message.txt create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/requires create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/00changelog.i create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/00manifest.i create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/data/m3.java.i create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/data/~2ehgtags.i create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/fncache create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/phaseroots create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/undo create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/undo.backup.fncache create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/undo.backupfiles create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/store/undo.phaseroots create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/undo.backup.dirstate create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/undo.bookmarks create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/undo.branch create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/undo.desc create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/undo.dirstate create mode 100755 rapidplugin/tests/resources/maven/hg/m3/.hg/wcache/checkisexec create mode 120000 rapidplugin/tests/resources/maven/hg/m3/.hg/wcache/checklink create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/wcache/checklink-target create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/wcache/checknoexec create mode 100644 rapidplugin/tests/resources/maven/hg/m3/.hg/wcache/manifestfulltextcache diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/00changelog.i b/rapidplugin/tests/resources/maven/hg/m3/.hg/00changelog.i new file mode 100644 index 0000000000000000000000000000000000000000..d3a8311050e54c57c5be7cfe169e60a95768812c GIT binary patch literal 57 zcmWN_K?=Yi3HXX‚Vè>«ýcý \ No newline at end of file diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/rbc-names-v1 b/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/rbc-names-v1 new file mode 100644 index 0000000..331d858 --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/rbc-names-v1 @@ -0,0 +1 @@ +default \ No newline at end of file diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/rbc-revs-v1 b/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/rbc-revs-v1 new file mode 100644 index 0000000000000000000000000000000000000000..69a325bbbae6b1ad19b112c4cd925c1a2d55160e GIT binary patch literal 16 Tcmd1VZrIKM1kYl_?to|jD;)$; literal 0 HcmV?d00001 diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/tags2-visible b/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/tags2-visible new file mode 100644 index 0000000..d16d86c --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/.hg/cache/tags2-visible @@ -0,0 +1,2 @@ +1 e65d56dc65e613a2501a6ab7d6903dd6b3aa7740 +6cb73187f2c9c3983baac1e38162bd1ea64d608f 1.0.0 diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/dirstate b/rapidplugin/tests/resources/maven/hg/m3/.hg/dirstate new file mode 100644 index 0000000000000000000000000000000000000000..cc6f7ea51ece398eb13db000bbfd761a798e84c4 GIT binary patch literal 88 zcmaDB8+Ir4ned_jsjThSCfHuvysF%R0Sm}uU}#(d6wivE|NAVEX3sU&%StRugo*3N RFJK4(iR)#gmn5ba0|0{$7mxq| literal 0 HcmV?d00001 diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/last-message.txt b/rapidplugin/tests/resources/maven/hg/m3/.hg/last-message.txt new file mode 100644 index 0000000..dc50929 --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/.hg/last-message.txt @@ -0,0 +1 @@ +Added tag 1.0.0 for changeset 6cb73187f2c9 \ No newline at end of file diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/requires b/rapidplugin/tests/resources/maven/hg/m3/.hg/requires new file mode 100644 index 0000000..0d88866 --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/.hg/requires @@ -0,0 +1,6 @@ +dotencode +fncache +generaldelta +revlogv1 +sparserevlog +store diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/store/00changelog.i b/rapidplugin/tests/resources/maven/hg/m3/.hg/store/00changelog.i new file mode 100644 index 0000000000000000000000000000000000000000..ff07e61546c6cd536961e53b30b2ced0844e6fad GIT binary patch literal 357 zcmZQzWME`~fGi-D31u_fqzXq`h*>d=vU%YCv?Uxj~wVnW?db zS)!>?l5wi3fw4u3xka*Zl1Wmsxn*KXszHi@p@F42S4e7ciGoMEf=x+=LkWpuCZQLVp$@C zg{851qM4b2iCJQrnMGo%iKStpp{bFHagvddxrK?jscEvgp_y?C7g#sQE(Qi0AWj0a z7$B|ySz0u2a@5zzy6sB5f7ZtH9d+=!JbS z4O{Unh@q+J(vmr6Qd%EPIwGVHI&J>UMH6Pvn7LdqCTPmc>2k8GCqzt%Tf-U3!*p=! uQPq_@t9~w3tzvahRkf^?@^ZL*sq0g*sL>_SsU~X=dS&m-Via1b&<6nFp*#}+ literal 0 HcmV?d00001 diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/store/data/~2ehgtags.i b/rapidplugin/tests/resources/maven/hg/m3/.hg/store/data/~2ehgtags.i new file mode 100644 index 0000000000000000000000000000000000000000..f8b4543d0dd423c3c424b8f79e8207f878587c3b GIT binary patch literal 112 zcmZQzW?*E100SVU4`nm{hXVJ>d#jO<^)G`0ZBzP5uj21ExyNl9SAh4K2*m qjFK&rjV&#VlM)k?4O5LR49$#^QVdfQ%}i3v3@p+V4D}5347dO`xE%uk literal 0 HcmV?d00001 diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/store/fncache b/rapidplugin/tests/resources/maven/hg/m3/.hg/store/fncache new file mode 100644 index 0000000..cd9b14c --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/.hg/store/fncache @@ -0,0 +1,2 @@ +data/m3.java.i +data/.hgtags.i diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/store/phaseroots b/rapidplugin/tests/resources/maven/hg/m3/.hg/store/phaseroots new file mode 100644 index 0000000..13081f8 --- /dev/null +++ b/rapidplugin/tests/resources/maven/hg/m3/.hg/store/phaseroots @@ -0,0 +1 @@ +1 6cb73187f2c9c3983baac1e38162bd1ea64d608f diff --git a/rapidplugin/tests/resources/maven/hg/m3/.hg/store/undo b/rapidplugin/tests/resources/maven/hg/m3/.hg/store/undo new file mode 100644 index 0000000000000000000000000000000000000000..e6285a67157a491b729fea22a53c2a15989217f8 GIT binary patch literal 52 zcmYdEEJ@VY%SbOtOfS~UWH8_|Fvv~J%S=lxE&*~44NZUo$r*`x>8UyS=^znv11Z^hv)dn(sHxF9f{fq}s=*H|wru`H3n!qV70(ag-i#4It*%px(> z#L_U)(A3DpILXM!+``1%)HK=L(9Af6t7zWjsIQTA+m(3#tc~Y8?0DqvepjGDNqQOS zC5h?93?}BO#s-NdrpBp;1_tIP7N$UGVPtA%n4D^1kY=2kn3R@cW}KG7MY7ERAJ04r literal 0 HcmV?d00001 From d0ed6dafa2194bf9c5c7c20c931528f06128d33e Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 3 Nov 2020 08:44:54 +0100 Subject: [PATCH 27/32] Reworked exception handling for hg checkout. --- rapidplugin/utils/utils.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 0c2309e..270e4b6 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -131,11 +131,17 @@ def hg_checkout(repo_path, version_tag, tmp_path): '-t', 'files', tmp_path ] - proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - out, err = proc.communicate() - os.chdir(wd) - if proc.returncode !=0: - raise Exception(str(err)) + try: + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + out, err = proc.communicate() + except Exception as e: + raise e + else: + if proc.returncode != 0: + raise Exception(str(err)) + finally: + os.chdir(wd) + class KafkaUtils: @staticmethod From 70c3700b4763c5c50e1f1e96560967af4f1fed8e Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 3 Nov 2020 10:59:54 +0100 Subject: [PATCH 28/32] Reworked exception handling for git checkout. --- rapidplugin/tests/test_utils.py | 7 ++++--- rapidplugin/utils/utils.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 4093dd6..221fe77 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -70,12 +70,13 @@ def test_checkout_hg(repo_path, repo_type, commit_tag, sources_dir, repos): @pytest.mark.parametrize('repo_path,repo_type,commit_tag', [("maven/git/m1", "git", "1.0.1"), - ("rapidplugin/tests/resources/maven/hg/m3", "hg", "1.0.1"), + ("maven/hg/m3", "hg", "1.0.1"), ("maven/svn/m2", "svn", "1.0.0")]) -def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir): +def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir, repos): + repo_path = os.path.join(repos, repo_path) with pytest.raises(Exception) as e: MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) - + print(str(e)+"\n") PAYLOAD_TAILOR_DATA = [ ({"product": "a"}, {"product": "a"}), diff --git a/rapidplugin/utils/utils.py b/rapidplugin/utils/utils.py index 270e4b6..8c144fd 100644 --- a/rapidplugin/utils/utils.py +++ b/rapidplugin/utils/utils.py @@ -106,7 +106,7 @@ def checkout_version(repo_path, repo_type, version_tag, base_dir): @staticmethod def git_checkout(repo_path, version_tag, tmp_path): repo = Repo(repo_path) - assert repo.tags[version_tag] is not None, "Tag: '{}' does not exist.".format(version_tag) + # assert repo.tag(version_tag) is None, "Tag: '{}' does not exist.".format(version_tag) archive_name = version_tag+".zip" archive_file_name = tmp_path/archive_name repo.git.archive(version_tag, o=archive_file_name) @@ -132,13 +132,14 @@ def hg_checkout(repo_path, version_tag, tmp_path): tmp_path ] try: - proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - out, err = proc.communicate() + proc_hg = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + out, err = proc_hg.communicate() except Exception as e: raise e else: - if proc.returncode != 0: - raise Exception(str(err)) + if proc_hg.returncode: + err_str = f"Cannot check out '{version_tag}' from repo '{repo_path}', [Error]" + str(err) + raise Exception(err_str) finally: os.chdir(wd) From f1ef589cca4ed1cacb4aba5a313ebcdd1732a891 Mon Sep 17 00:00:00 2001 From: cg122 Date: Tue, 3 Nov 2020 11:30:22 +0100 Subject: [PATCH 29/32] Added test case of hg repo in integration testing. --- integration_tests/test_plugin.py | 22 +++++++++++----------- rapidplugin/tests/test_utils.py | 1 - 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/integration_tests/test_plugin.py b/integration_tests/test_plugin.py index 6f67d72..afaca5d 100644 --- a/integration_tests/test_plugin.py +++ b/integration_tests/test_plugin.py @@ -101,16 +101,16 @@ def plugin_run(mock_in, mock_out, mock_log, mock_err, # "repoType": "svn", # "commitTag": "1.0.0" # }, - # { - # "forge": "mvn", - # "groupId": "test-mvn", - # "artifactId": "m3", - # "version": "1.0.0", - # "sourcesUrl": "", - # "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/hg/m3", - # "repoType": "hg", - # "commitTag": "1.0.0" - # }, + { + "forge": "mvn", + "groupId": "test-mvn", + "artifactId": "m3", + "version": "1.0.0", + "sourcesUrl": "", + "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/hg/m3", + "repoType": "hg", + "commitTag": "1.0.0" + }, { "forge": "debian", "product": "d1", @@ -160,7 +160,7 @@ def test_successes(plugin_run, in_message): "artifactId": "m3", "version": "1.0.1", "sourcesUrl": "", - "repoPath": "maven/hg/m3", + "repoPath": "/home/plugin/rapidplugin/tests/resources/maven/hg/m3", "repoType": "hg", "commitTag": "1.0.1" }]) diff --git a/rapidplugin/tests/test_utils.py b/rapidplugin/tests/test_utils.py index 221fe77..b87820d 100644 --- a/rapidplugin/tests/test_utils.py +++ b/rapidplugin/tests/test_utils.py @@ -76,7 +76,6 @@ def test_checkout_fail(repo_path, repo_type, commit_tag, sources_dir, repos): repo_path = os.path.join(repos, repo_path) with pytest.raises(Exception) as e: MavenUtils.checkout_version(repo_path, repo_type, commit_tag, sources_dir) - print(str(e)+"\n") PAYLOAD_TAILOR_DATA = [ ({"product": "a"}, {"product": "a"}), From 27e3f861797b178189818ba567e998cf08033041 Mon Sep 17 00:00:00 2001 From: MagielBruntink Date: Tue, 3 Nov 2020 13:19:43 +0100 Subject: [PATCH 30/32] Changed the running of units tests to happen within Docker container. --- Dockerfile | 5 +++++ tox.ini | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 16e7aab..60a7504 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,9 @@ RUN useradd -m plugin USER plugin +RUN git config --global user.email "research@softwareimprovementgroup.com" \ + && git config --global user.name "Software Improvement Group Research" + WORKDIR /home/plugin COPY --chown=plugin rapidplugin rapidplugin/ @@ -25,4 +28,6 @@ COPY --chown=plugin README.md . RUN python -m pip install -r requirements.txt +RUN python -m pytest rapidplugin/tests + ENTRYPOINT ["python", "/home/plugin/entrypoint.py"] diff --git a/tox.ini b/tox.ini index 0d9a494..f625791 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,6 @@ allowlist_externals=python* commands = flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - pytest rapidplugin/tests docker build -t rapidplugin-test . docker-compose -f integration_tests/kafka-compose.yml up -d docker-compose -f integration_tests/kafka-compose.yml run wait-for-kafka From 646881a663d87c08ff04d96a5af2ef0d1ee12ab7 Mon Sep 17 00:00:00 2001 From: MagielBruntink Date: Tue, 3 Nov 2020 13:21:03 +0100 Subject: [PATCH 31/32] Changed Github action to not run units tests outside of Docker container. --- .github/workflows/python-package.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 98b15ca..692ddb3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -36,9 +36,6 @@ jobs: run: | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Run unit tests - run: | - pytest rapidplugin/tests - name: Build docker image run: | From 2e243a000d8a5cce27ee0f702b753b4c9b50c70b Mon Sep 17 00:00:00 2001 From: MagielBruntink Date: Tue, 3 Nov 2020 13:29:21 +0100 Subject: [PATCH 32/32] Changed unit test running to happen after Docker image building. --- .github/workflows/python-package.yml | 4 ++++ Dockerfile | 2 -- tox.ini | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 692ddb3..fed3772 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -41,6 +41,10 @@ jobs: run: | docker build -t rapidplugin-test . + - name: Run unit tests + run: | + docker run --entrypoint python rapidplugin-test -m pytest rapidplugin/tests + - name: Setup integration environment run: | docker-compose -f integration_tests/kafka-compose.yml up -d diff --git a/Dockerfile b/Dockerfile index 60a7504..5291175 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,4 @@ COPY --chown=plugin README.md . RUN python -m pip install -r requirements.txt -RUN python -m pytest rapidplugin/tests - ENTRYPOINT ["python", "/home/plugin/entrypoint.py"] diff --git a/tox.ini b/tox.ini index f625791..2b6d46c 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ commands = flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics docker build -t rapidplugin-test . + docker run --entrypoint python rapidplugin-test -m pytest rapidplugin/tests docker-compose -f integration_tests/kafka-compose.yml up -d docker-compose -f integration_tests/kafka-compose.yml run wait-for-kafka pytest integration_tests