Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update validate and versions, refactor metadata and urls #305

Merged
merged 7 commits into from
Mar 8, 2024
Merged

Update validate and versions, refactor metadata and urls #305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ea9a191
Add and refactor metadata and urls code and tests #247
johnmhoran Feb 24, 2024
e8c9abb
Refactor warnings, header construction, purl normalization and relate…
johnmhoran Feb 28, 2024
b7edacc
Merge branch 'main' into 247-purlcli-update-validate-and-versions #247
johnmhoran Feb 28, 2024
bd3b2ca
Add logging, finish data structure update and refactor code and tests…
johnmhoran Mar 5, 2024
4020e1b
Address PR comments #247
johnmhoran Mar 7, 2024
888d8d8
Update app.log path #247
johnmhoran Mar 7, 2024
9d293e8
Rename log file and convert location to constant #247
johnmhoran Mar 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
507 changes: 314 additions & 193 deletions purldb-toolkit/src/purldb_toolkit/purlcli.py
View file
Open in desktop

Large diffs are not rendered by default.

154 changes: 154 additions & 0 deletions purldb-toolkit/tests/data/purlcli/expected_metadata_output.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
"--purl": [
"pkg:pypi/fetchcode",
"pkg:pypi/fetchcode@0.3.0",
"pkg:pypi/fetchcode@0.3.0?os=windows",
"pkg:pypi/fetchcode@0.3.0os=windows",
"pkg:pypi/fetchcode@5.0.0",
"pkg:cargo/banquo",
"pkg:nginx/nginx",
"pkg:gem/rails",
Expand All @@ -19,13 +22,18 @@
"purls": [
"pkg:pypi/fetchcode",
"pkg:pypi/fetchcode@0.3.0",
"pkg:pypi/fetchcode@0.3.0?os=windows",
"pkg:pypi/fetchcode@0.3.0os=windows",
"pkg:pypi/fetchcode@5.0.0",
"pkg:cargo/banquo",
"pkg:nginx/nginx",
"pkg:gem/rails",
"pkg:rubygems/rails"
],
"errors": [],
"warnings": [
"'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo",
"'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo",
"'pkg:nginx/nginx' not supported with `metadata` command",
"'pkg:gem/rails' not supported with `metadata` command"
]
Expand Down Expand Up @@ -320,6 +328,152 @@
"repository_download_url": null,
"api_data_url": null
},
{
"purl": "pkg:pypi/fetchcode@0.3.0?os=windows",
"type": "pypi",
"namespace": null,
"name": "fetchcode",
"version": "0.3.0",
"qualifiers": {
"os": "windows"
},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [],
"keywords": [],
"homepage_url": "https://github.com/nexB/fetchcode",
"download_url": null,
"api_url": "https://pypi.org/pypi/fetchcode/json",
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"copyright": null,
"license_expression": null,
"declared_license": "Apache-2.0",
"notice_text": null,
"root_path": null,
"dependencies": [],
"contains_source_code": null,
"source_packages": [],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null
},
{
"purl": "pkg:pypi/fetchcode@0.1.0",
"type": "pypi",
"namespace": null,
"name": "fetchcode",
"version": "0.1.0",
"qualifiers": {},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [],
"keywords": [],
"homepage_url": "https://github.com/nexB/fetchcode",
"download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl",
"api_url": "https://pypi.org/pypi/fetchcode/json",
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"copyright": null,
"license_expression": null,
"declared_license": "Apache-2.0",
"notice_text": null,
"root_path": null,
"dependencies": [],
"contains_source_code": null,
"source_packages": [],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null
},
{
"purl": "pkg:pypi/fetchcode@0.2.0",
"type": "pypi",
"namespace": null,
"name": "fetchcode",
"version": "0.2.0",
"qualifiers": {},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [],
"keywords": [],
"homepage_url": "https://github.com/nexB/fetchcode",
"download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl",
"api_url": "https://pypi.org/pypi/fetchcode/json",
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"copyright": null,
"license_expression": null,
"declared_license": "Apache-2.0",
"notice_text": null,
"root_path": null,
"dependencies": [],
"contains_source_code": null,
"source_packages": [],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null
},
{
"purl": "pkg:pypi/fetchcode@0.3.0",
"type": "pypi",
"namespace": null,
"name": "fetchcode",
"version": "0.3.0",
"qualifiers": {},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [],
"keywords": [],
"homepage_url": "https://github.com/nexB/fetchcode",
"download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl",
"api_url": "https://pypi.org/pypi/fetchcode/json",
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"copyright": null,
"license_expression": null,
"declared_license": "Apache-2.0",
"notice_text": null,
"root_path": null,
"dependencies": [],
"contains_source_code": null,
"source_packages": [],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null
},
{
"purl": "pkg:cargo/banquo",
"type": "cargo",
Expand Down
9 changes: 9 additions & 0 deletions purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
"--purl": [
"pkg:pypi/fetchcode",
"pkg:pypi/fetchcode@0.3.0",
"pkg:pypi/fetchcode@0.3.0?os=windows",
"pkg:pypi/fetchcode@0.3.0os=windows",
"pkg:pypi/fetchcode@5.0.0",
"pkg:cargo/banquo",
"pkg:nginx/nginx",
"pkg:gem/rails",
Expand All @@ -20,6 +23,9 @@
"purls": [
"pkg:pypi/fetchcode",
"pkg:pypi/fetchcode@0.3.0",
"pkg:pypi/fetchcode@0.3.0?os=windows",
"pkg:pypi/fetchcode@0.3.0os=windows",
"pkg:pypi/fetchcode@5.0.0",
"pkg:cargo/banquo",
"pkg:nginx/nginx",
"pkg:gem/rails",
Expand All @@ -28,6 +34,9 @@
"errors": [],
"warnings": [
"input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'",
"input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'",
"input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'",
"input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'",
"'pkg:nginx/nginx' not supported with `metadata` command",
"'pkg:gem/rails' not supported with `metadata` command"
]
Expand Down
56 changes: 3 additions & 53 deletions purldb-toolkit/tests/data/purlcli/expected_urls_output.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,12 @@
"pkg:gem/bundler-sass",
"pkg:rubygems/bundler-sass",
"pkg:pypi/matchcode",
"pkg:rubygems/bundler-sass",
"abcdefg",
"pkg/abc",
"pkg:nuget/auth0-aspnet@1.1.0",
"pkg:cargo/socksprox"
"pkg:nuget/auth0-aspnet@1.1.0"
],
"--file": null,
"--output": "/mnt/c/nexb/purldb-testing/2024-current-01-testing/json-output/2024-02-14-urls--output-01.json"
"--output": "<stdout>"
},
"purls": [
"pkg:pypi/fetchcode",
Expand All @@ -50,11 +48,9 @@
"pkg:gem/bundler-sass",
"pkg:rubygems/bundler-sass",
"pkg:pypi/matchcode",
"pkg:rubygems/bundler-sass",
"abcdefg",
"pkg/abc",
"pkg:nuget/auth0-aspnet@1.1.0",
"pkg:cargo/socksprox"
"pkg:nuget/auth0-aspnet@1.1.0"
],
"errors": [],
"warnings": [
Expand Down Expand Up @@ -330,29 +326,6 @@
"url": "https://rubygems.org/gems/bundler-sass"
}
},
{
"purl": "pkg:rubygems/bundler-sass",
"download_url": {
"url": null
},
"inferred_urls": [
{
"url": "https://rubygems.org/gems/bundler-sass"
}
],
"repo_download_url": {
"url": null
},
"repo_download_url_by_package_type": {
"url": null
},
"repo_url": {
"url": "https://rubygems.org/gems/bundler-sass"
},
"url": {
"url": "https://rubygems.org/gems/bundler-sass"
}
},
{
"purl": "pkg:nuget/auth0-aspnet@1.1.0",
"download_url": {
Expand All @@ -378,29 +351,6 @@
"url": {
"url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0"
}
},
{
"purl": "pkg:cargo/socksprox",
"download_url": {
"url": null
},
"inferred_urls": [
{
"url": "https://crates.io/crates/socksprox"
}
],
"repo_download_url": {
"url": null
},
"repo_download_url_by_package_type": {
"url": null
},
"repo_url": {
"url": "https://crates.io/crates/socksprox"
},
"url": {
"url": "https://crates.io/crates/socksprox"
}
}
]
}
Loading
Loading