From e1ad340c2d802b1d64e5bd256ae40a21f0bbf03c Mon Sep 17 00:00:00 2001 From: Christopher Angelo Phillips <32073428+spiffcs@users.noreply.github.com> Date: Fri, 20 Oct 2023 11:29:13 -0400 Subject: [PATCH] fix: allow packages to be captured from DIST/EGG case (#2239) * fix: allow packages to be captured from DIST/EGG case Signed-off-by: Christopher Phillips * test: update expected glob paths Signed-off-by: Christopher Phillips --------- Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/python/cataloger.go | 9 +- syft/pkg/cataloger/python/cataloger_test.go | 79 +++++++++++ .../casesensitive/DIST-INFO/METADATA | 47 ++++++ .../casesensitive/DIST-INFO/RECORD | 6 + .../casesensitive/DIST-INFO/direct_url.json | 1 + .../casesensitive/DIST-INFO/top_level.txt | 2 + .../casesensitive/EGG-INFO/PKG-INFO | 134 ++++++++++++++++++ .../casesensitive/EGG-INFO/PKG-INFO-INVALID | 69 +++++++++ .../casesensitive/EGG-INFO/RECORD | 6 + .../casesensitive/EGG-INFO/top_level.txt | 1 + .../site-packages/v.DIST-INFO/METADATA | 1 + .../site-packages/w.EGG-INFO/PKG-INFO | 1 + 12 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD create mode 100644 syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt create mode 100644 syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/v.DIST-INFO/METADATA create mode 100644 syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/w.EGG-INFO/PKG-INFO diff --git a/syft/pkg/cataloger/python/cataloger.go b/syft/pkg/cataloger/python/cataloger.go index 88868255e819..af5f0bd319e4 100644 --- a/syft/pkg/cataloger/python/cataloger.go +++ b/syft/pkg/cataloger/python/cataloger.go @@ -29,5 +29,12 @@ func NewPythonIndexCataloger(cfg CatalogerConfig) *generic.Cataloger { // NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories. func NewPythonPackageCataloger() *generic.Cataloger { return generic.NewCataloger("python-package-cataloger"). - WithParserByGlobs(parseWheelOrEgg, eggInfoGlob, "**/*dist-info/METADATA", "**/*egg-info/PKG-INFO") + WithParserByGlobs( + parseWheelOrEgg, + eggInfoGlob, + "**/*dist-info/METADATA", + "**/*egg-info/PKG-INFO", + "**/*DIST-INFO/METADATA", + "**/*EGG-INFO/PKG-INFO", + ) } diff --git a/syft/pkg/cataloger/python/cataloger_test.go b/syft/pkg/cataloger/python/cataloger_test.go index b37f6724c4dd..a6e84290ee76 100644 --- a/syft/pkg/cataloger/python/cataloger_test.go +++ b/syft/pkg/cataloger/python/cataloger_test.go @@ -69,6 +69,43 @@ func Test_PackageCataloger(t *testing.T) { }, }, }, + { + name: "egg-info directory case sensitive", + fixtures: []string{ + "test-fixtures/casesensitive/EGG-INFO/PKG-INFO", + "test-fixtures/casesensitive/EGG-INFO/RECORD", + "test-fixtures/casesensitive/EGG-INFO/top_level.txt", + }, + expectedPackage: pkg.Package{ + Name: "requests", + Version: "2.22.0", + PURL: "pkg:pypi/requests@2.22.0", + Type: pkg.PythonPkg, + Language: pkg.Python, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/casesensitive/EGG-INFO/PKG-INFO")), + ), + FoundBy: "python-package-cataloger", + MetadataType: pkg.PythonPackageMetadataType, + Metadata: pkg.PythonPackageMetadata{ + Name: "requests", + Version: "2.22.0", + Platform: "UNKNOWN", + Author: "Kenneth Reitz", + AuthorEmail: "me@kennethreitz.org", + SitePackagesRootPath: "test-fixtures/casesensitive", + Files: []pkg.PythonFileRecord{ + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, + {Path: "requests/__pycache__/utils.cpython-38.pyc"}, + {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + }, + TopLevelPackages: []string{"requests"}, + }, + }, + }, { name: "dist-info directory", fixtures: []string{ @@ -109,6 +146,46 @@ func Test_PackageCataloger(t *testing.T) { }, }, }, + { + name: "dist-info directory case sensitive", + fixtures: []string{ + "test-fixtures/casesensitive/DIST-INFO/METADATA", + "test-fixtures/casesensitive/DIST-INFO/RECORD", + "test-fixtures/casesensitive/DIST-INFO/top_level.txt", + "test-fixtures/casesensitive/DIST-INFO/direct_url.json", + }, + expectedPackage: pkg.Package{ + Name: "Pygments", + Version: "2.6.1", + PURL: "pkg:pypi/Pygments@2.6.1?vcs_url=git+https://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Type: pkg.PythonPkg, + Language: pkg.Python, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/casesensitive/DIST-INFO/METADATA")), + ), + FoundBy: "python-package-cataloger", + MetadataType: pkg.PythonPackageMetadataType, + Metadata: pkg.PythonPackageMetadata{ + Name: "Pygments", + Version: "2.6.1", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + SitePackagesRootPath: "test-fixtures/casesensitive", + Files: []pkg.PythonFileRecord{ + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "Pygments-2.6.1.dist-info/RECORD"}, + {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + + {Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"}, + }, + TopLevelPackages: []string{"pygments", "something_else"}, + DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }, + }, + }, { name: "malformed-record", fixtures: []string{ @@ -278,6 +355,8 @@ func Test_PackageCataloger_Globs(t *testing.T) { name: "obtain index files", fixture: "test-fixtures/glob-paths", expected: []string{ + "site-packages/v.DIST-INFO/METADATA", + "site-packages/w.EGG-INFO/PKG-INFO", "site-packages/x.dist-info/METADATA", "site-packages/y.egg-info/PKG-INFO", "site-packages/z.egg-info", diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA new file mode 100644 index 000000000000..924780dfd45a --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA @@ -0,0 +1,47 @@ +Metadata-Version: 2.1 +Name: Pygments +Version: 2.6.1 +Summary: Pygments is a syntax highlighting package written in Python. +Home-page: https://pygments.org/ +Author: Georg Brandl +Author-email: georg@python.org +License: BSD License +Keywords: syntax highlighting +Platform: any +Classifier: License :: OSI Approved :: BSD License +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: System Administrators +Classifier: Development Status :: 6 - Mature +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Operating System :: OS Independent +Classifier: Topic :: Text Processing :: Filters +Classifier: Topic :: Utilities +Requires-Python: >=3.5 + + +Pygments +~~~~~~~~ + +Pygments is a syntax highlighting package written in Python. + +It is a generic syntax highlighter suitable for use in code hosting, forums, +wikis or other applications that need to prettify source code. Highlights +are: + +* a wide range of over 500 languages and other text formats is supported +* special attention is paid to details, increasing quality by a fair amount +* support for new languages and formats are added easily +* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences +* it is usable as a command-line tool and as a library + +:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. +:license: BSD, see LICENSE for details. + diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD new file mode 100644 index 000000000000..b7beac37eba7 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD @@ -0,0 +1,6 @@ +../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220 +Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449 +Pygments-2.6.1.dist-info/RECORD,, +pygments/__pycache__/__init__.cpython-38.pyc,, +pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778 +pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778 diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json new file mode 100644 index 000000000000..5d42a36e3114 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json @@ -0,0 +1 @@ +{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}} \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt new file mode 100644 index 000000000000..2c30fc0c4c0e --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt @@ -0,0 +1,2 @@ +pygments +something_else \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO new file mode 100644 index 000000000000..a73770668150 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO @@ -0,0 +1,134 @@ +Metadata-Version: 2.1 +Name: requests +Version: 2.22.0 +Summary: Python HTTP for Humans. +Home-page: http://python-requests.org +Author: Kenneth Reitz +Author-email: me@kennethreitz.org +License: Apache 2.0 +Description: Requests: HTTP for Humans™ + ========================== + + [![image](https://img.shields.io/pypi/v/requests.svg)](https://pypi.org/project/requests/) + [![image](https://img.shields.io/pypi/l/requests.svg)](https://pypi.org/project/requests/) + [![image](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests/) + [![codecov.io](https://codecov.io/github/requests/requests/coverage.svg?branch=master)](https://codecov.io/github/requests/requests) + [![image](https://img.shields.io/github/contributors/requests/requests.svg)](https://github.com/requests/requests/graphs/contributors) + [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz) + + Requests is the only *Non-GMO* HTTP library for Python, safe for human + consumption. + + ![image](https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg) + + Behold, the power of Requests: + + ``` {.sourceCode .python} + >>> import requests + >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) + >>> r.status_code + 200 + >>> r.headers['content-type'] + 'application/json; charset=utf8' + >>> r.encoding + 'utf-8' + >>> r.text + u'{"type":"User"...' + >>> r.json() + {u'disk_usage': 368627, u'private_gists': 484, ...} + ``` + + See [the similar code, sans Requests](https://gist.github.com/973705). + + [![image](https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png)](http://docs.python-requests.org/) + + Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, + without the need for manual labor. There's no need to manually add query + strings to your URLs, or to form-encode your POST data. Keep-alive and + HTTP connection pooling are 100% automatic, thanks to + [urllib3](https://github.com/shazow/urllib3). + + Besides, all the cool kids are doing it. Requests is one of the most + downloaded Python packages of all time, pulling in over 11,000,000 + downloads every month. You don't want to be left out! + + Feature Support + --------------- + + Requests is ready for today's web. + + - International Domains and URLs + - Keep-Alive & Connection Pooling + - Sessions with Cookie Persistence + - Browser-style SSL Verification + - Basic/Digest Authentication + - Elegant Key/Value Cookies + - Automatic Decompression + - Automatic Content Decoding + - Unicode Response Bodies + - Multipart File Uploads + - HTTP(S) Proxy Support + - Connection Timeouts + - Streaming Downloads + - `.netrc` Support + - Chunked Requests + + Requests officially supports Python 2.7 & 3.4–3.7, and runs great on + PyPy. + + Installation + ------------ + + To install Requests, simply use [pipenv](http://pipenv.org/) (or pip, of + course): + + ``` {.sourceCode .bash} + $ pipenv install requests + ✨🍰✨ + ``` + + Satisfaction guaranteed. + + Documentation + ------------- + + Fantastic documentation is available at + , for a limited time only. + + How to Contribute + ----------------- + + 1. Become more familiar with the project by reading our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/) and our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/). + 2. Check for open issues or open a fresh issue to start a discussion + around a feature idea or a bug. There is a [Contributor + Friendly](https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open) + tag for issues that should be ideal for people who are not very + familiar with the codebase yet. + 3. Fork [the repository](https://github.com/requests/requests) on + GitHub to start making your changes to the **master** branch (or + branch off of it). + 4. Write a test which shows that the bug was fixed or that the feature + works as expected. + 5. Send a pull request and bug the maintainer until it gets merged and + published. :) Make sure to add yourself to + [AUTHORS](https://github.com/requests/requests/blob/master/AUTHORS.rst). + + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* +Description-Content-Type: text/markdown +Provides-Extra: security +Provides-Extra: socks \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID new file mode 100644 index 000000000000..173d8c5126b2 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID @@ -0,0 +1,69 @@ +Metadata-Version: 1.1 +Name: mxnet +Version: 1.8.0 +Summary: + + + + + + + + + + + + + + + + +MXNet Python Package +==================== +This directory and nested files contain MXNet Python package and language binding. + +## Installation +To install MXNet Python package, visit MXNet [Install Instruction](https://mxnet.apache.org/get_started) + + +## Running the unit tests + +For running unit tests, you will need the [nose PyPi package](https://pypi.python.org/pypi/nose). To install: +```bash +pip install --upgrade nose +``` + +Once ```nose``` is installed, run the following from MXNet root directory (please make sure the installation path of ```nosetests``` is included in your ```$PATH``` environment variable): +``` +nosetests tests/python/unittest +nosetests tests/python/train + +``` + +Home-page: https://github.com/apache/incubator-mxnet +Author: UNKNOWN +Author-email: UNKNOWN +License: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: C++ +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Other +Classifier: Programming Language :: Perl +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Topic :: Scientific/Engineering +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Classifier: Topic :: Scientific/Engineering :: Mathematics +Classifier: Topic :: Software Development +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development :: Libraries :: Python Modules diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD new file mode 100644 index 000000000000..a2c2f97c3534 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD @@ -0,0 +1,6 @@ +requests-2.22.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +requests/__init__.py,sha256=PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA,3921 +requests/__pycache__/__version__.cpython-38.pyc,, +requests/__pycache__/utils.cpython-38.pyc,, +requests/__version__.py,sha256=Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc,436 +requests/utils.py,sha256=LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A,30049 diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt new file mode 100644 index 000000000000..663bd1f6a2ae --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/v.DIST-INFO/METADATA b/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/v.DIST-INFO/METADATA new file mode 100644 index 000000000000..5ffba7b57dc7 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/v.DIST-INFO/METADATA @@ -0,0 +1 @@ +bogus diff --git a/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/w.EGG-INFO/PKG-INFO b/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/w.EGG-INFO/PKG-INFO new file mode 100644 index 000000000000..5ffba7b57dc7 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/w.EGG-INFO/PKG-INFO @@ -0,0 +1 @@ +bogus