From a07bfe7dfa02551236a944115a6a72ec85f16a8a Mon Sep 17 00:00:00 2001 From: Shane Dell <32347414+Shanedell@users.noreply.github.com> Date: Thu, 27 Apr 2023 09:04:30 -0400 Subject: [PATCH] Create python requirements metadata (#1759) - Create new metadata struct and type for python requirements. - Update parsing of python requirements to use python requirements metadata. - Remove extras and url from line. Add them to metadata instead. - Add unit test to test that extras are removed from package name. - Update test to look at requirements metadata. - Will need updated in future to support more than just == for the version constraint. - Update JSON schema data Closes anchore/grype#1246 Closes anchore/grype#1251 Signed-off-by: Shane Dell --- internal/constants.go | 2 +- schema/json/generate.go | 57 +- schema/json/schema-7.1.5.json | 1810 +++++++++++++++++ .../snapshot/TestDirectoryEncoder.golden | 4 - .../TestEncodeFullJSONDocument.golden | 4 - .../snapshot/TestImageEncoder.golden | 20 +- .../stereoscope-fixture-image-simple.golden | Bin 15360 -> 15360 bytes syft/pkg/cataloger/python/package.go | 17 + .../cataloger/python/parse_requirements.go | 101 +- .../python/parse_requirements_test.go | 167 +- .../test-fixtures/requires/requirements.txt | 5 +- syft/pkg/metadata.go | 3 + syft/pkg/python_requirements_metadata.go | 9 + 13 files changed, 2110 insertions(+), 89 deletions(-) create mode 100644 schema/json/schema-7.1.5.json create mode 100644 syft/pkg/python_requirements_metadata.go diff --git a/internal/constants.go b/internal/constants.go index 2390e6da83f..5adf784b7a0 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -6,5 +6,5 @@ const ( // JSONSchemaVersion is the current schema version output by the JSON encoder // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "7.1.4" + JSONSchemaVersion = "7.1.5" ) diff --git a/schema/json/generate.go b/schema/json/generate.go index db5d1adbedb..9197c4d3fe7 100644 --- a/schema/json/generate.go +++ b/schema/json/generate.go @@ -30,34 +30,35 @@ can be extended to include specific package metadata struct shapes in the future // TODO: this should be generated from reflection of whats in the pkg package type artifactMetadataContainer struct { - Alpm pkg.AlpmMetadata - Apk pkg.ApkMetadata - Binary pkg.BinaryMetadata - Cocopods pkg.CocoapodsMetadata - Conan pkg.ConanMetadata - ConanLock pkg.ConanLockMetadata - Dart pkg.DartPubMetadata - Dotnet pkg.DotnetDepsMetadata - Dpkg pkg.DpkgMetadata - Gem pkg.GemMetadata - GoBin pkg.GolangBinMetadata - GoMod pkg.GolangModMetadata - Hackage pkg.HackageMetadata - Java pkg.JavaMetadata - KbPackage pkg.KbPackageMetadata - LinuxKernel pkg.LinuxKernelMetadata - LinuxKernelModule pkg.LinuxKernelModuleMetadata - Nix pkg.NixStoreMetadata - NpmPackage pkg.NpmPackageJSONMetadata - NpmPackageLock pkg.NpmPackageLockJSONMetadata - MixLock pkg.MixLockMetadata - Php pkg.PhpComposerJSONMetadata - Portage pkg.PortageMetadata - PythonPackage pkg.PythonPackageMetadata - PythonPipfilelock pkg.PythonPipfileLockMetadata - Rebar pkg.RebarLockMetadata - Rpm pkg.RpmMetadata - RustCargo pkg.CargoPackageMetadata + Alpm pkg.AlpmMetadata + Apk pkg.ApkMetadata + Binary pkg.BinaryMetadata + Cocopods pkg.CocoapodsMetadata + Conan pkg.ConanMetadata + ConanLock pkg.ConanLockMetadata + Dart pkg.DartPubMetadata + Dotnet pkg.DotnetDepsMetadata + Dpkg pkg.DpkgMetadata + Gem pkg.GemMetadata + GoBin pkg.GolangBinMetadata + GoMod pkg.GolangModMetadata + Hackage pkg.HackageMetadata + Java pkg.JavaMetadata + KbPackage pkg.KbPackageMetadata + LinuxKernel pkg.LinuxKernelMetadata + LinuxKernelModule pkg.LinuxKernelModuleMetadata + Nix pkg.NixStoreMetadata + NpmPackage pkg.NpmPackageJSONMetadata + NpmPackageLock pkg.NpmPackageLockJSONMetadata + MixLock pkg.MixLockMetadata + Php pkg.PhpComposerJSONMetadata + Portage pkg.PortageMetadata + PythonPackage pkg.PythonPackageMetadata + PythonPipfilelock pkg.PythonPipfileLockMetadata + PythonRequirements pkg.PythonRequirementsMetadata + Rebar pkg.RebarLockMetadata + Rpm pkg.RpmMetadata + RustCargo pkg.CargoPackageMetadata } func main() { diff --git a/schema/json/schema-7.1.5.json b/schema/json/schema-7.1.5.json new file mode 100644 index 00000000000..16ff4dd9082 --- /dev/null +++ b/schema/json/schema-7.1.5.json @@ -0,0 +1,1810 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/anchore/syft/syft/formats/syftjson/model/document", + "$ref": "#/$defs/Document", + "$defs": { + "AlpmFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "type": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "gid": { + "type": "string" + }, + "time": { + "type": "string", + "format": "date-time" + }, + "size": { + "type": "string" + }, + "link": { + "type": "string" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object" + }, + "AlpmMetadata": { + "properties": { + "basepackage": { + "type": "string" + }, + "package": { + "type": "string" + }, + "version": { + "type": "string" + }, + "description": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "packager": { + "type": "string" + }, + "license": { + "type": "string" + }, + "url": { + "type": "string" + }, + "validation": { + "type": "string" + }, + "reason": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + }, + "backup": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "basepackage", + "package", + "version", + "description", + "architecture", + "size", + "packager", + "license", + "url", + "validation", + "reason", + "files", + "backup" + ] + }, + "ApkFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "ownerUid": { + "type": "string" + }, + "ownerGid": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "ApkMetadata": { + "properties": { + "package": { + "type": "string" + }, + "originPackage": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "version": { + "type": "string" + }, + "license": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "url": { + "type": "string" + }, + "description": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "installedSize": { + "type": "integer" + }, + "pullDependencies": { + "items": { + "type": "string" + }, + "type": "array" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array" + }, + "pullChecksum": { + "type": "string" + }, + "gitCommitOfApkPort": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/ApkFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "originPackage", + "maintainer", + "version", + "license", + "architecture", + "url", + "description", + "size", + "installedSize", + "pullDependencies", + "provides", + "pullChecksum", + "gitCommitOfApkPort", + "files" + ] + }, + "BinaryMetadata": { + "properties": { + "matches": { + "items": { + "$ref": "#/$defs/ClassifierMatch" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "matches" + ] + }, + "CargoPackageMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string" + }, + "checksum": { + "type": "string" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "checksum", + "dependencies" + ] + }, + "ClassifierMatch": { + "properties": { + "classifier": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Location" + } + }, + "type": "object", + "required": [ + "classifier", + "location" + ] + }, + "CocoapodsMetadata": { + "properties": { + "checksum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "checksum" + ] + }, + "ConanLockMetadata": { + "properties": { + "ref": { + "type": "string" + }, + "package_id": { + "type": "string" + }, + "prev": { + "type": "string" + }, + "requires": { + "type": "string" + }, + "build_requires": { + "type": "string" + }, + "py_requires": { + "type": "string" + }, + "options": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "path": { + "type": "string" + }, + "context": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "ConanMetadata": { + "properties": { + "ref": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "Coordinates": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "DartPubMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "hosted_url": { + "type": "string" + }, + "vcs_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Descriptor": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "configuration": true + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Digest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "Document": { + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/Package" + }, + "type": "array" + }, + "artifactRelationships": { + "items": { + "$ref": "#/$defs/Relationship" + }, + "type": "array" + }, + "files": { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + "secrets": { + "items": { + "$ref": "#/$defs/Secrets" + }, + "type": "array" + }, + "source": { + "$ref": "#/$defs/Source" + }, + "distro": { + "$ref": "#/$defs/LinuxRelease" + }, + "descriptor": { + "$ref": "#/$defs/Descriptor" + }, + "schema": { + "$ref": "#/$defs/Schema" + } + }, + "type": "object", + "required": [ + "artifacts", + "artifactRelationships", + "source", + "distro", + "descriptor", + "schema" + ] + }, + "DotnetDepsMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "path": { + "type": "string" + }, + "sha512": { + "type": "string" + }, + "hashPath": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "path", + "sha512", + "hashPath" + ] + }, + "DpkgFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "isConfigFile": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "path", + "isConfigFile" + ] + }, + "DpkgMetadata": { + "properties": { + "package": { + "type": "string" + }, + "source": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/DpkgFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ] + }, + "File": { + "properties": { + "id": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Coordinates" + }, + "metadata": { + "$ref": "#/$defs/FileMetadataEntry" + }, + "contents": { + "type": "string" + }, + "digests": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "id", + "location" + ] + }, + "FileMetadataEntry": { + "properties": { + "mode": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "linkDestination": { + "type": "string" + }, + "userID": { + "type": "integer" + }, + "groupID": { + "type": "integer" + }, + "mimeType": { + "type": "string" + }, + "size": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "mode", + "type", + "userID", + "groupID", + "mimeType", + "size" + ] + }, + "GemMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "type": "array" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "GolangBinMetadata": { + "properties": { + "goBuildSettings": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "goCompiledVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "h1Digest": { + "type": "string" + }, + "mainModule": { + "type": "string" + } + }, + "type": "object", + "required": [ + "goCompiledVersion", + "architecture" + ] + }, + "GolangModMetadata": { + "properties": { + "h1Digest": { + "type": "string" + } + }, + "type": "object" + }, + "HackageMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "snapshotURL": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "IDLikes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "JavaManifest": { + "properties": { + "main": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "namedSections": { + "patternProperties": { + ".*": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "JavaMetadata": { + "properties": { + "virtualPath": { + "type": "string" + }, + "manifest": { + "$ref": "#/$defs/JavaManifest" + }, + "pomProperties": { + "$ref": "#/$defs/PomProperties" + }, + "pomProject": { + "$ref": "#/$defs/PomProject" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "virtualPath" + ] + }, + "KbPackageMetadata": { + "properties": { + "product_id": { + "type": "string" + }, + "kb": { + "type": "string" + } + }, + "type": "object", + "required": [ + "product_id", + "kb" + ] + }, + "LinuxKernelMetadata": { + "properties": { + "name": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "version": { + "type": "string" + }, + "extendedVersion": { + "type": "string" + }, + "buildTime": { + "type": "string" + }, + "author": { + "type": "string" + }, + "format": { + "type": "string" + }, + "rwRootFS": { + "type": "boolean" + }, + "swapDevice": { + "type": "integer" + }, + "rootDevice": { + "type": "integer" + }, + "videoMode": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "architecture", + "version" + ] + }, + "LinuxKernelModuleMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "path": { + "type": "string" + }, + "description": { + "type": "string" + }, + "author": { + "type": "string" + }, + "license": { + "type": "string" + }, + "kernelVersion": { + "type": "string" + }, + "versionMagic": { + "type": "string" + }, + "parameters": { + "patternProperties": { + ".*": { + "$ref": "#/$defs/LinuxKernelModuleParameter" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "LinuxKernelModuleParameter": { + "properties": { + "type": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "type": "object" + }, + "LinuxRelease": { + "properties": { + "prettyName": { + "type": "string" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "idLike": { + "$ref": "#/$defs/IDLikes" + }, + "version": { + "type": "string" + }, + "versionID": { + "type": "string" + }, + "versionCodename": { + "type": "string" + }, + "buildID": { + "type": "string" + }, + "imageID": { + "type": "string" + }, + "imageVersion": { + "type": "string" + }, + "variant": { + "type": "string" + }, + "variantID": { + "type": "string" + }, + "homeURL": { + "type": "string" + }, + "supportURL": { + "type": "string" + }, + "bugReportURL": { + "type": "string" + }, + "privacyPolicyURL": { + "type": "string" + }, + "cpeName": { + "type": "string" + }, + "supportEnd": { + "type": "string" + } + }, + "type": "object" + }, + "Location": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + }, + "annotations": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "MixLockMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "NixStoreMetadata": { + "properties": { + "outputHash": { + "type": "string" + }, + "output": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "outputHash", + "files" + ] + }, + "NpmPackageJSONMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + }, + "private": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "licenses", + "homepage", + "description", + "url", + "private" + ] + }, + "NpmPackageLockJSONMetadata": { + "properties": { + "resolved": { + "type": "string" + }, + "integrity": { + "type": "string" + } + }, + "type": "object", + "required": [ + "resolved", + "integrity" + ] + }, + "Package": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "foundBy": { + "type": "string" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "language": { + "type": "string" + }, + "cpes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "purl": { + "type": "string" + }, + "metadataType": { + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/AlpmMetadata" + }, + { + "$ref": "#/$defs/ApkMetadata" + }, + { + "$ref": "#/$defs/BinaryMetadata" + }, + { + "$ref": "#/$defs/CargoPackageMetadata" + }, + { + "$ref": "#/$defs/CocoapodsMetadata" + }, + { + "$ref": "#/$defs/ConanLockMetadata" + }, + { + "$ref": "#/$defs/ConanMetadata" + }, + { + "$ref": "#/$defs/DartPubMetadata" + }, + { + "$ref": "#/$defs/DotnetDepsMetadata" + }, + { + "$ref": "#/$defs/DpkgMetadata" + }, + { + "$ref": "#/$defs/GemMetadata" + }, + { + "$ref": "#/$defs/GolangBinMetadata" + }, + { + "$ref": "#/$defs/GolangModMetadata" + }, + { + "$ref": "#/$defs/HackageMetadata" + }, + { + "$ref": "#/$defs/JavaMetadata" + }, + { + "$ref": "#/$defs/KbPackageMetadata" + }, + { + "$ref": "#/$defs/LinuxKernelMetadata" + }, + { + "$ref": "#/$defs/LinuxKernelModuleMetadata" + }, + { + "$ref": "#/$defs/MixLockMetadata" + }, + { + "$ref": "#/$defs/NixStoreMetadata" + }, + { + "$ref": "#/$defs/NpmPackageJSONMetadata" + }, + { + "$ref": "#/$defs/NpmPackageLockJSONMetadata" + }, + { + "$ref": "#/$defs/PhpComposerJSONMetadata" + }, + { + "$ref": "#/$defs/PortageMetadata" + }, + { + "$ref": "#/$defs/PythonPackageMetadata" + }, + { + "$ref": "#/$defs/PythonPipfileLockMetadata" + }, + { + "$ref": "#/$defs/PythonRequirementsMetadata" + }, + { + "$ref": "#/$defs/RebarLockMetadata" + }, + { + "$ref": "#/$defs/RpmMetadata" + } + ] + } + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "foundBy", + "locations", + "licenses", + "language", + "cpes", + "purl" + ] + }, + "PhpComposerAuthors": { + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "PhpComposerExternalReference": { + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": "string" + }, + "reference": { + "type": "string" + }, + "shasum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "type", + "url", + "reference" + ] + }, + "PhpComposerJSONMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "require": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "provide": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "require-dev": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "suggest": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "type": { + "type": "string" + }, + "notification-url": { + "type": "string" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array" + }, + "description": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array" + }, + "time": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ] + }, + "PomParent": { + "properties": { + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "type": "object", + "required": [ + "groupId", + "artifactId", + "version" + ] + }, + "PomProject": { + "properties": { + "path": { + "type": "string" + }, + "parent": { + "$ref": "#/$defs/PomParent" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "groupId", + "artifactId", + "version", + "name" + ] + }, + "PomProperties": { + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "extraFields": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path", + "name", + "groupId", + "artifactId", + "version" + ] + }, + "PortageFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PortageMetadata": { + "properties": { + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/PortageFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "installedSize", + "files" + ] + }, + "PythonDirectURLOriginInfo": { + "properties": { + "url": { + "type": "string" + }, + "commitId": { + "type": "string" + }, + "vcs": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "PythonFileDigest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "PythonFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/PythonFileDigest" + }, + "size": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PythonPackageMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "license": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorEmail": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/PythonFileRecord" + }, + "type": "array" + }, + "sitePackagesRootPath": { + "type": "string" + }, + "topLevelPackages": { + "items": { + "type": "string" + }, + "type": "array" + }, + "directUrlOrigin": { + "$ref": "#/$defs/PythonDirectURLOriginInfo" + } + }, + "type": "object", + "required": [ + "name", + "version", + "license", + "author", + "authorEmail", + "platform", + "sitePackagesRootPath" + ] + }, + "PythonPipfileLockMetadata": { + "properties": { + "hashes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "index": { + "type": "string" + } + }, + "type": "object", + "required": [ + "hashes", + "index" + ] + }, + "PythonRequirementsMetadata": { + "properties": { + "name": { + "type": "string" + }, + "extras": { + "items": { + "type": "string" + }, + "type": "array" + }, + "versionConstraint": { + "type": "string" + }, + "url": { + "type": "string" + }, + "markers": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "name", + "extras", + "versionConstraint", + "url", + "markers" + ] + }, + "RebarLockMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "Relationship": { + "properties": { + "parent": { + "type": "string" + }, + "child": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": true + }, + "type": "object", + "required": [ + "parent", + "child", + "type" + ] + }, + "RpmMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "license": { + "type": "string" + }, + "vendor": { + "type": "string" + }, + "modularityLabel": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/RpmdbFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "license", + "vendor", + "modularityLabel", + "files" + ] + }, + "RpmdbFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "mode": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "userName": { + "type": "string" + }, + "groupName": { + "type": "string" + }, + "flags": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "mode", + "size", + "digest", + "userName", + "groupName", + "flags" + ] + }, + "Schema": { + "properties": { + "version": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "version", + "url" + ] + }, + "SearchResult": { + "properties": { + "classification": { + "type": "string" + }, + "lineNumber": { + "type": "integer" + }, + "lineOffset": { + "type": "integer" + }, + "seekPosition": { + "type": "integer" + }, + "length": { + "type": "integer" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "classification", + "lineNumber", + "lineOffset", + "seekPosition", + "length" + ] + }, + "Secrets": { + "properties": { + "location": { + "$ref": "#/$defs/Coordinates" + }, + "secrets": { + "items": { + "$ref": "#/$defs/SearchResult" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "location", + "secrets" + ] + }, + "Source": { + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "target": true + }, + "type": "object", + "required": [ + "id", + "type", + "target" + ] + } + } +} diff --git a/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden b/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden index 3288035324b..5b03f3e9eeb 100644 --- a/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden +++ b/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden @@ -87,9 +87,5 @@ "configuration": { "config-key": "config-value" } - }, - "schema": { - "version": "7.1.4", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.4.json" } } diff --git a/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden b/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden index 322e8cb0fde..c8305e5b381 100644 --- a/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden +++ b/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden @@ -187,9 +187,5 @@ "configuration": { "config-key": "config-value" } - }, - "schema": { - "version": "7.1.4", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.4.json" } } diff --git a/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden b/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden index eda15743f5f..93c74ea4441 100644 --- a/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden +++ b/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden @@ -9,7 +9,7 @@ "locations": [ { "path": "/somefile-1.txt", - "layerID": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59" + "layerID": "sha256:7e139310bd6ce0956d65a70d26a6d31b240a4f47094a831638f05d381b6c424a" } ], "licenses": [ @@ -40,7 +40,7 @@ "locations": [ { "path": "/somefile-2.txt", - "layerID": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec" + "layerID": "sha256:cc833bf31a480c064d65ca67ee37f77f0d0c8ab98eedde7b286ad1ef6f5bdcac" } ], "licenses": [], @@ -64,11 +64,11 @@ ], "artifactRelationships": [], "source": { - "id": "1a678f111c8ddc66fd82687bb024e0dd6af61314404937a80e810c0cf317b796", + "id": "0af8fa79f5497297e4e32f3e03de14ac20ad695159df0ac8373e6543614b9a50", "type": "image", "target": { "userInput": "user-image-input", - "imageID": "sha256:3c51b06feb0cda8ee62d0e3755ef2a8496a6b71f8a55b245f07f31c4bb813d31", + "imageID": "sha256:0cb4395791986bda17562bd6f76811bb6f163f686e198397197ef8241bed58df", "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "tags": [ @@ -78,17 +78,17 @@ "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59", + "digest": "sha256:7e139310bd6ce0956d65a70d26a6d31b240a4f47094a831638f05d381b6c424a", "size": 22 }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec", + "digest": "sha256:cc833bf31a480c064d65ca67ee37f77f0d0c8ab98eedde7b286ad1ef6f5bdcac", "size": 16 } ], - "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NzMsImRpZ2VzdCI6InNoYTI1NjozYzUxYjA2ZmViMGNkYThlZTYyZDBlMzc1NWVmMmE4NDk2YTZiNzFmOGE1NWIyNDVmMDdmMzFjNGJiODEzZDMxIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmYjZiZWVjYjc1YjM5ZjRiYjgxM2RiZjE3N2U1MDFlZGQ1ZGRiM2U2OWJiNDVjZWRlYjc4YzY3NmVlMWI3YTU5In0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjMxOWI1ODhjZTY0MjUzYTg3YjUzM2M4ZWQwMWNmMDAyNWUwZWFjOThlN2I1MTZlMTI1MzI5NTdlMTI0NGZkZWMifV19", - "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjItMDgtMDFUMjA6MDk6MjIuNTA5NDIxNzEyWiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIyLTA4LTAxVDIwOjA5OjIyLjQ4Nzg5NTUxOVoiLCJjcmVhdGVkX2J5IjoiQUREIGZpbGUtMS50eHQgL3NvbWVmaWxlLTEudHh0ICMgYnVpbGRraXQiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9LHsiY3JlYXRlZCI6IjIwMjItMDgtMDFUMjA6MDk6MjIuNTA5NDIxNzEyWiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6ZmI2YmVlY2I3NWIzOWY0YmI4MTNkYmYxNzdlNTAxZWRkNWRkYjNlNjliYjQ1Y2VkZWI3OGM2NzZlZTFiN2E1OSIsInNoYTI1NjozMTliNTg4Y2U2NDI1M2E4N2I1MzNjOGVkMDFjZjAwMjVlMGVhYzk4ZTdiNTE2ZTEyNTMyOTU3ZTEyNDRmZGVjIl19fQ==", + "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NzEsImRpZ2VzdCI6InNoYTI1NjowY2I0Mzk1NzkxOTg2YmRhMTc1NjJiZDZmNzY4MTFiYjZmMTYzZjY4NmUxOTgzOTcxOTdlZjgyNDFiZWQ1OGRmIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1Njo3ZTEzOTMxMGJkNmNlMDk1NmQ2NWE3MGQyNmE2ZDMxYjI0MGE0ZjQ3MDk0YTgzMTYzOGYwNWQzODFiNmM0MjRhIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OmNjODMzYmYzMWE0ODBjMDY0ZDY1Y2E2N2VlMzdmNzdmMGQwYzhhYjk4ZWVkZGU3YjI4NmFkMWVmNmY1YmRjYWMifV19", + "config": "eyJhcmNoaXRlY3R1cmUiOiJhcm02NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjMtMDQtMThUMTQ6MDk6NDIuMzAxMDI2MzhaIiwiaGlzdG9yeSI6W3siY3JlYXRlZCI6IjIwMjMtMDQtMThUMTQ6MDk6NDIuMjg3OTQyNzEzWiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0xLnR4dCAvc29tZWZpbGUtMS50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn0seyJjcmVhdGVkIjoiMjAyMy0wNC0xOFQxNDowOTo0Mi4zMDEwMjYzOFoiLCJjcmVhdGVkX2J5IjoiQUREIGZpbGUtMi50eHQgL3NvbWVmaWxlLTIudHh0ICMgYnVpbGRraXQiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9XSwib3MiOiJsaW51eCIsInJvb3RmcyI6eyJ0eXBlIjoibGF5ZXJzIiwiZGlmZl9pZHMiOlsic2hhMjU2OjdlMTM5MzEwYmQ2Y2UwOTU2ZDY1YTcwZDI2YTZkMzFiMjQwYTRmNDcwOTRhODMxNjM4ZjA1ZDM4MWI2YzQyNGEiLCJzaGEyNTY6Y2M4MzNiZjMxYTQ4MGMwNjRkNjVjYTY3ZWUzN2Y3N2YwZDBjOGFiOThlZWRkZTdiMjg2YWQxZWY2ZjViZGNhYyJdfX0=", "repoDigests": [], "architecture": "", "os": "" @@ -110,9 +110,5 @@ "configuration": { "config-key": "config-value" } - }, - "schema": { - "version": "7.1.4", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.4.json" } } diff --git a/syft/formats/syftjson/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/syft/formats/syftjson/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden index 11a1958c8935029370bb6a48ad18899403776949..9455ea930505e90c2e18f1f311a3ec9f5b3b49a4 100644 GIT binary patch literal 15360 zcmeHOZBN@s5YFfO6*=EmlI;7gBi#qCAhl9;R4vz&u0m+NFW@7wBipG!i2r_LCxs+E z2*DwxVk0E7-ksUW%+7e`@fewe(U{8^$4VqtW61?ctngAOjFUuoET|VsI9NbqiDT)! zA`Bp%dXb-U-R>znPQ9NX%PvIyLLDfG%d7hS4UR;4_*K4SVk})F^)0%9)!D#}N(f+|f z*QX=b!~N>A>JG|$>=wqqb(31Si?nJKG9J6E0;+Ep`!+WhuBZw6Ur}_kw@#2n@&8{A zvLeT9yrCSR=ez_tM`<>_1j&m$uRL71j;gCkZR3%?azzOvP5WMDNYDOnt3Ks%(S8q9F#}gQiT&?7-1T?&N&&QC`~-Ynki%; z0stqYg>;TeFQrEo8Ksk0IcKet2~k2@?7Z+iu|}Kd}cdEC`m8xT1!;hcd9sT)`9oF3c!TfaD{sf}D4EO(sbh@oM+;5%vY)h2D=Lcw@ zwg0!z|KWy%SNtCd0r7t)C-arZ|LZ8NlkK+0pw5qPGk2&FQ4eAGJ4R^O{56?(o>e-{ zTyc6=TfEK7D%0aSDFqe>`4|@5jsGfLT_?cIepP96{;9H1Mhv|km2Tl$K837&^`a|) zbbj%;f@FXG{b=H{Wj@X5C^I0$d`(mF)CI|2^kzJ-P4=R9V|_Lk^m9I%jvb`Uvqei) z4M=;KRmIgLPwQ%7wqmKw*^A!m#Es_UQ1mQh>$}c)+Zuk*)Dr!=hQ9Z+=KjyV%l`-# zhz0+@SD{;+Sgv@t$asO(8v7$C>6`e^A?zcBI|yURNU;Bc3ZLz31u#kjfppJx*@eJd z0me~2+jfC*&_K{Y&_K|@3(l@red}5IqIU*%-;$4-!8QLa`$NM|LyTV%<*^6 z|GDJB|95d}=J;RRV*sGnL>IHi<82`&;B|*3uImD*E7kxiHc;gXsQ6QWtXpa;*&L;< zdrvlnY47Qj`GM$d`v}q2@t+C*<)Qzpy=@=E;THdGI*lGfvH6wzc@Vq zeX6=QqJ!r7H}g*a2yNqC5U8s@x=DVl&t_fal`C9cntbAVKE148GU%msBTcVN$CHs8 zsygd493!O?Yl&oDY8AUUcF;B)BM%M8hQ&@2?+x@EOC*hjmr}qiVwklJ2ks@`^_SAo!|C77k@TM0W{9lOR|2sJX m*HO8~L+kfOT07~mwKGLGH!stJc4DTmNzg#hK+wR}HSiykm4wCs literal 15360 zcmeHOZExE)5ccQ&3Qzl**nFpCU>~}sKnoO0(Pka6qA2)sOtjjPA<0F9Apdm%%?PJlu^wTR3f}9JyT$INc2+m0LY2Bo z+kCjL@($GhDlwJ2$Qy9Ytb$~en2A>+IZ9kvg#0`iT_ul-cgg5g^6bIcA**Z}Z~(B(HF&Ar5m|{4vYNJ{c9W@%S2tt-v&Z z<%AK&j!Ak<=oup;l8qP}NRkOgHD&)`*NePv%IXqzyV}xCq&m%{q*>c-Il>!@J(C9~ zC;gC*;g}AZ_f3CTmlIei|JJvyc5m}$Zjs3ZiUxaqxH<5pdka}WC)5Q(m;?gGWhr0tP6t%jdrI6?%$4P|rP|4?pM&h_=5YAJNDFF!+e z=O#b-`rq0AFCRUB@#OT+yZo?a|3~-pb^kM>h>ZLHT{hiS9UXLze0HTul=BrV(0Tsv zApVzZWBq4Da24nOAzp{8ByWT5uCD>cd{a{`qhswgO%cpt?3XzyKzi#9rAlcoiC`%g zUKpNQNsP1tFy!MF(O;Tr$E;^4>mDcoaY|$I0;)H+h~;rvH-(wBfi4O=DJR(Aw)|HO zE&9Nk2Tfz##f!!xI2i>qu3_a~Im1wY^Cnc-b$RowK?Je9KAl3bE@xgkEgUMbY%|AE zEzCJe9!{2s=O}qHG3QG~UzX$91TYj|G+M(-sO_VosV=8w-a?Q?i#70clsq*S#>?(4 zl)6E1->^M)ge&iF&=!2&*P@?*$}3RoyMrrTC;o7bBnK^r^2y(p_)mNk|Fby_6#1V| zT6A|6H-yKWXV{(GRY_a?k3>Mb{}ZMtG5}Qf2&I}M{yRuy`jg|oIoZYf?JkPDcm3 z?))!NY%~7@`QP~c=OFLQlK<`c8X%c=%h}RQ6HMD^GdBokZnS^%#EZz(ejsi)nA)#M zAa-^9-uOQf`JY2905|;KJOOGyBW~ya)JOaurVB>?Z$Iu>liFYw-8=)Xe_`qy8gQ*i zEJO=L3q%V<3;ci&PqIn?_abld!Ax4Gf-TyUKK3XcKB_mIfGV=884#+q6MM_z9S3#2P@x!egFUf diff --git a/syft/pkg/cataloger/python/package.go b/syft/pkg/cataloger/python/package.go index 0cb1e3f71ab..d247597dc0a 100644 --- a/syft/pkg/cataloger/python/package.go +++ b/syft/pkg/cataloger/python/package.go @@ -40,6 +40,23 @@ func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPip return p } +func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.PythonRequirementsMetadata, locations ...source.Location) pkg.Package { + p := pkg.Package{ + Name: name, + Version: version, + Locations: source.NewLocationSet(locations...), + PURL: packageURL(name, version, nil), + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: metadata, + } + + p.SetID() + + return p +} + func newPackageForPackage(m pkg.PythonPackageMetadata, sources ...source.Location) pkg.Package { var licenses []string if m.License != "" { diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index 9b6e5db5835..c2b5a122a0a 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -3,6 +3,7 @@ package python import ( "bufio" "fmt" + "regexp" "strings" "unicode" @@ -15,6 +16,11 @@ import ( var _ generic.Parser = parseRequirementsTxt +var ( + extrasRegex = regexp.MustCompile(`\[.*\]`) + urlRegex = regexp.MustCompile("@.*git.*") +) + // parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a // specific version. func parseRequirementsTxt(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { @@ -23,6 +29,7 @@ func parseRequirementsTxt(_ source.FileResolver, _ *generic.Environment, reader scanner := bufio.NewScanner(reader) for scanner.Scan() { line := scanner.Text() + rawLineNoComments := removeTrailingComment(line) line = trimRequirementsTxtLine(line) if line == "" { @@ -57,15 +64,25 @@ func parseRequirementsTxt(_ source.FileResolver, _ *generic.Environment, reader return !unicode.IsLetter(r) && !unicode.IsNumber(r) }) + // TODO: Update to support more than only == + versionConstraint := fmt.Sprintf("== %s", version) + if name == "" || version == "" { log.WithFields("path", reader.RealPath).Debugf("found empty package in requirements.txt line: %q", line) continue } packages = append( packages, - newPackageForIndex( + newPackageForRequirementsWithMetadata( name, version, + pkg.PythonRequirementsMetadata{ + Name: name, + Extras: parseExtras(rawLineNoComments), + VersionConstraint: versionConstraint, + URL: parseURL(rawLineNoComments), + Markers: parseMarkers(rawLineNoComments), + }, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), ) @@ -93,6 +110,7 @@ func trimRequirementsTxtLine(line string) string { line = strings.TrimSpace(line) line = removeTrailingComment(line) line = removeEnvironmentMarkers(line) + line = checkForRegex(line) // remove extras and url from line if found return line } @@ -121,3 +139,84 @@ func removeEnvironmentMarkers(line string) string { return parts[0] } + +func parseExtras(packageName string) []string { + if extrasRegex.MatchString(packageName) { + // Remove square brackets + extras := strings.TrimFunc(extrasRegex.FindString(packageName), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsNumber(r) + }) + + // Remove any additional whitespace + extras = strings.ReplaceAll(extras, " ", "") + + return strings.Split(extras, ",") + } + + return []string{} +} + +func parseMarkers(line string) map[string]string { + markers := map[string]string{} + parts := strings.SplitN(line, ";", 2) + + if len(parts) == 2 { + splittableMarkers := parts[1] + + for _, combineString := range []string{" or ", " and "} { + splittableMarkers = strings.TrimSpace( + strings.ReplaceAll(splittableMarkers, combineString, ","), + ) + } + + splittableMarkers = strings.TrimSpace(splittableMarkers) + + for _, mark := range strings.Split(splittableMarkers, ",") { + markparts := strings.Split(mark, " ") + markers[markparts[0]] = strings.Join(markparts[1:], " ") + } + } + + return markers +} + +func parseURL(line string) string { + parts := strings.Split(line, "@") + + if len(parts) > 1 { + desiredIndex := -1 + + for index, part := range parts { + part := strings.TrimFunc(part, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsNumber(r) + }) + + if strings.HasPrefix(part, "git") { + desiredIndex = index + break + } + } + + if desiredIndex != -1 { + return strings.TrimSpace(strings.Join(parts[desiredIndex:], "@")) + } + } + + return "" +} + +// function to check a string for all possilbe regex expressions, replacing it if found +func checkForRegex(stringToCheck string) string { + stringToReturn := stringToCheck + + for _, r := range []*regexp.Regexp{ + urlRegex, + extrasRegex, + } { + if r.MatchString(stringToCheck) { + stringToReturn = r.ReplaceAllString(stringToCheck, "") + } + } + + return stringToReturn +} diff --git a/syft/pkg/cataloger/python/parse_requirements_test.go b/syft/pkg/cataloger/python/parse_requirements_test.go index 9131fcf96af..b25179c5070 100644 --- a/syft/pkg/cataloger/python/parse_requirements_test.go +++ b/syft/pkg/cataloger/python/parse_requirements_test.go @@ -14,44 +14,135 @@ func TestParseRequirementsTxt(t *testing.T) { locations := source.NewLocationSet(source.NewLocation(fixture)) expectedPkgs := []pkg.Package{ { - Name: "flask", - Version: "4.0.0", - PURL: "pkg:pypi/flask@4.0.0", - Locations: locations, - Language: pkg.Python, - Type: pkg.PythonPkg, - }, - { - Name: "foo", - Version: "1.0.0", - PURL: "pkg:pypi/foo@1.0.0", - Locations: locations, - Language: pkg.Python, - Type: pkg.PythonPkg, - }, - { - Name: "SomeProject", - Version: "5.4", - PURL: "pkg:pypi/SomeProject@5.4", - Locations: locations, - Language: pkg.Python, - Type: pkg.PythonPkg, - }, - { - Name: "argh", - Version: "0.26.2", - PURL: "pkg:pypi/argh@0.26.2", - Locations: locations, - Language: pkg.Python, - Type: pkg.PythonPkg, - }, - { - Name: "argh", - Version: "0.26.3", - PURL: "pkg:pypi/argh@0.26.3", - Locations: locations, - Language: pkg.Python, - Type: pkg.PythonPkg, + Name: "flask", + Version: "4.0.0", + PURL: "pkg:pypi/flask@4.0.0", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "flask", + Extras: []string{}, + VersionConstraint: "== 4.0.0", + URL: "", + Markers: map[string]string{}, + }, + }, + { + Name: "foo", + Version: "1.0.0", + PURL: "pkg:pypi/foo@1.0.0", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "foo", + Extras: []string{}, + VersionConstraint: "== 1.0.0", + URL: "", + Markers: map[string]string{}, + }, + }, + { + Name: "SomeProject", + Version: "5.4", + PURL: "pkg:pypi/SomeProject@5.4", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "SomeProject", + Extras: []string{}, + VersionConstraint: "== 5.4", + URL: "", + Markers: map[string]string{"python_version": "< '3.8'"}, + }, + }, + { + Name: "argh", + Version: "0.26.2", + PURL: "pkg:pypi/argh@0.26.2", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "argh", + Extras: []string{}, + VersionConstraint: "== 0.26.2", + URL: "", + Markers: map[string]string{}, + }, + }, + { + Name: "argh", + Version: "0.26.3", + PURL: "pkg:pypi/argh@0.26.3", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "argh", + Extras: []string{}, + VersionConstraint: "== 0.26.3", + URL: "", + Markers: map[string]string{}, + }, + }, + { + Name: "celery", + Version: "4.4.7", + PURL: "pkg:pypi/celery@4.4.7", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "celery", + Extras: []string{"redis", "pytest"}, + VersionConstraint: "== 4.4.7", + URL: "", + Markers: map[string]string{}, + }, + }, + { + Name: "requests", + Version: "2.8", + PURL: "pkg:pypi/requests@2.8", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "requests", + Extras: []string{"security"}, + VersionConstraint: "== 2.8", + URL: "", + Markers: map[string]string{ + "python_version": `< "2.7"`, + "sys_platform": `== "linux"`, + }, + }, + }, + { + Name: "GithubSampleProject", + Version: "3.7.1", + PURL: "pkg:pypi/GithubSampleProject@3.7.1", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonRequirementsMetadataType, + Metadata: pkg.PythonRequirementsMetadata{ + Name: "GithubSampleProject", + Extras: []string{}, + VersionConstraint: "== 3.7.1", + URL: "git+https://github.com/owner/repo@releases/tag/v3.7.1", + Markers: map[string]string{}, + }, }, } diff --git a/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt b/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt index c172f7c3bc9..df96fe539d8 100644 --- a/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt +++ b/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt @@ -16,4 +16,7 @@ argh==0.26.2 \ argh==0.26.3 --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65 # CommentedOut == 1.2.3 # maybe invalid, but found out in the wild -==2.3.4 \ No newline at end of file +==2.3.4 +celery[redis, pytest] == 4.4.7 # should remove [redis, pytest] +requests[security] == 2.8.* ; python_version < "2.7" and sys_platform == "linux" +GithubSampleProject == 3.7.1 @ git+https://github.com/owner/repo@releases/tag/v3.7.1 diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go index d6b8e14e0d4..c6d4a036ec6 100644 --- a/syft/pkg/metadata.go +++ b/syft/pkg/metadata.go @@ -36,6 +36,7 @@ const ( PortageMetadataType MetadataType = "PortageMetadata" PythonPackageMetadataType MetadataType = "PythonPackageMetadata" PythonPipfileLockMetadataType MetadataType = "PythonPipfileLockMetadata" + PythonRequirementsMetadataType MetadataType = "PythonRequirementsMetadata" RebarLockMetadataType MetadataType = "RebarLockMetadataType" RpmMetadataType MetadataType = "RpmMetadata" RustCargoPackageMetadataType MetadataType = "RustCargoPackageMetadata" @@ -67,6 +68,7 @@ var AllMetadataTypes = []MetadataType{ PortageMetadataType, PythonPackageMetadataType, PythonPipfileLockMetadataType, + PythonRequirementsMetadataType, RebarLockMetadataType, RpmMetadataType, RustCargoPackageMetadataType, @@ -98,6 +100,7 @@ var MetadataTypeByName = map[MetadataType]reflect.Type{ PortageMetadataType: reflect.TypeOf(PortageMetadata{}), PythonPackageMetadataType: reflect.TypeOf(PythonPackageMetadata{}), PythonPipfileLockMetadataType: reflect.TypeOf(PythonPipfileLockMetadata{}), + PythonRequirementsMetadataType: reflect.TypeOf(PythonRequirementsMetadata{}), RebarLockMetadataType: reflect.TypeOf(RebarLockMetadata{}), RpmMetadataType: reflect.TypeOf(RpmMetadata{}), RustCargoPackageMetadataType: reflect.TypeOf(CargoPackageMetadata{}), diff --git a/syft/pkg/python_requirements_metadata.go b/syft/pkg/python_requirements_metadata.go new file mode 100644 index 00000000000..da675e6aa0e --- /dev/null +++ b/syft/pkg/python_requirements_metadata.go @@ -0,0 +1,9 @@ +package pkg + +type PythonRequirementsMetadata struct { + Name string `json:"name" mapstruct:"Name"` + Extras []string `json:"extras" mapstruct:"Extras"` + VersionConstraint string `json:"versionConstraint" mapstruct:"VersionConstraint"` + URL string `json:"url" mapstruct:"URL"` + Markers map[string]string `json:"markers" mapstruct:"Markers"` +}