From 98de2e2f6205b1660f98915cbed22695821fa9c8 Mon Sep 17 00:00:00 2001 From: Asi Greenholts <88270351+asi-cider@users.noreply.github.com> Date: Fri, 2 Feb 2024 21:53:42 +0200 Subject: [PATCH] Adding metadata fields when parsing yarn.lock and poetry.lock (#2350) * Adding the resolved and integrity fields of yarn.lock to the parsed metadata. This addition is similar to the metadata added when parsing package-lock.json. Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * fix comment Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * Adding the Index field to metadeta when parsing poetry.lock similarly to the existing Pipfile metadata Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * fixing struct accoding to tests Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * remove old schema change Signed-off-by: Alex Goodman * remove empty constants Signed-off-by: Alex Goodman * re-generate JSON schema Signed-off-by: Alex Goodman * update document ref Signed-off-by: Alex Goodman * fix linting Signed-off-by: Alex Goodman --------- Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> Signed-off-by: Alex Goodman Co-authored-by: Alex Goodman --- internal/constants.go | 2 +- schema/json/schema-16.0.2.json | 2252 +++++++++++++++++ syft/internal/packagemetadata/generated.go | 2 + syft/internal/packagemetadata/names.go | 2 + syft/pkg/cataloger/javascript/package.go | 4 +- .../cataloger/javascript/parse_yarn_lock.go | 69 +- .../javascript/parse_yarn_lock_test.go | 50 +- syft/pkg/cataloger/python/package.go | 2 +- .../cataloger/python/parse_pipfile_lock.go | 2 +- .../pkg/cataloger/python/parse_poetry_lock.go | 29 +- .../python/parse_poetry_lock_test.go | 4 + .../python/test-fixtures/poetry/poetry.lock | 5 + syft/pkg/npm.go | 6 + syft/pkg/python.go | 5 + 14 files changed, 2398 insertions(+), 36 deletions(-) create mode 100644 schema/json/schema-16.0.2.json diff --git a/internal/constants.go b/internal/constants.go index fc55f9e3129..ce391e1640e 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -3,5 +3,5 @@ package internal const ( // JSONSchemaVersion is the current schema version output by the JSON encoder // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "16.0.1" + JSONSchemaVersion = "16.0.2" ) diff --git a/schema/json/schema-16.0.2.json b/schema/json/schema-16.0.2.json new file mode 100644 index 00000000000..c18e688d5ae --- /dev/null +++ b/schema/json/schema-16.0.2.json @@ -0,0 +1,2252 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "anchore.io/schema/syft/json/16.0.2/document", + "$ref": "#/$defs/Document", + "$defs": { + "AlpmDbEntry": { + "properties": { + "basepackage": { + "type": "string" + }, + "package": { + "type": "string" + }, + "version": { + "type": "string" + }, + "description": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "packager": { + "type": "string" + }, + "url": { + "type": "string" + }, + "validation": { + "type": "string" + }, + "reason": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + }, + "backup": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "basepackage", + "package", + "version", + "description", + "architecture", + "size", + "packager", + "url", + "validation", + "reason", + "files", + "backup" + ] + }, + "AlpmFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "type": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "gid": { + "type": "string" + }, + "time": { + "type": "string", + "format": "date-time" + }, + "size": { + "type": "string" + }, + "link": { + "type": "string" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object" + }, + "ApkDbEntry": { + "properties": { + "package": { + "type": "string" + }, + "originPackage": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "version": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "url": { + "type": "string" + }, + "description": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "installedSize": { + "type": "integer" + }, + "pullDependencies": { + "items": { + "type": "string" + }, + "type": "array" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array" + }, + "pullChecksum": { + "type": "string" + }, + "gitCommitOfApkPort": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/ApkFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "originPackage", + "maintainer", + "version", + "architecture", + "url", + "description", + "size", + "installedSize", + "pullDependencies", + "provides", + "pullChecksum", + "gitCommitOfApkPort", + "files" + ] + }, + "ApkFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "ownerUid": { + "type": "string" + }, + "ownerGid": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "BinarySignature": { + "properties": { + "matches": { + "items": { + "$ref": "#/$defs/ClassifierMatch" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "matches" + ] + }, + "CConanFileEntry": { + "properties": { + "ref": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "CConanInfoEntry": { + "properties": { + "ref": { + "type": "string" + }, + "package_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "CConanLockEntry": { + "properties": { + "ref": { + "type": "string" + }, + "package_id": { + "type": "string" + }, + "prev": { + "type": "string" + }, + "requires": { + "items": { + "type": "string" + }, + "type": "array" + }, + "build_requires": { + "items": { + "type": "string" + }, + "type": "array" + }, + "py_requires": { + "items": { + "type": "string" + }, + "type": "array" + }, + "options": { + "$ref": "#/$defs/KeyValues" + }, + "path": { + "type": "string" + }, + "context": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "CPE": { + "properties": { + "cpe": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "type": "object", + "required": [ + "cpe" + ] + }, + "ClassifierMatch": { + "properties": { + "classifier": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Location" + } + }, + "type": "object", + "required": [ + "classifier", + "location" + ] + }, + "CocoaPodfileLockEntry": { + "properties": { + "checksum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "checksum" + ] + }, + "Coordinates": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "DartPubspecLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "hosted_url": { + "type": "string" + }, + "vcs_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Descriptor": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "configuration": true + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Digest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "Document": { + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/Package" + }, + "type": "array" + }, + "artifactRelationships": { + "items": { + "$ref": "#/$defs/Relationship" + }, + "type": "array" + }, + "files": { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + "source": { + "$ref": "#/$defs/Source" + }, + "distro": { + "$ref": "#/$defs/LinuxRelease" + }, + "descriptor": { + "$ref": "#/$defs/Descriptor" + }, + "schema": { + "$ref": "#/$defs/Schema" + } + }, + "type": "object", + "required": [ + "artifacts", + "artifactRelationships", + "source", + "distro", + "descriptor", + "schema" + ] + }, + "DotnetDepsEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "path": { + "type": "string" + }, + "sha512": { + "type": "string" + }, + "hashPath": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "path", + "sha512", + "hashPath" + ] + }, + "DotnetPortableExecutableEntry": { + "properties": { + "assemblyVersion": { + "type": "string" + }, + "legalCopyright": { + "type": "string" + }, + "comments": { + "type": "string" + }, + "internalName": { + "type": "string" + }, + "companyName": { + "type": "string" + }, + "productName": { + "type": "string" + }, + "productVersion": { + "type": "string" + } + }, + "type": "object", + "required": [ + "assemblyVersion", + "legalCopyright", + "companyName", + "productName", + "productVersion" + ] + }, + "DpkgDbEntry": { + "properties": { + "package": { + "type": "string" + }, + "source": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "installedSize": { + "type": "integer" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array" + }, + "preDepends": { + "items": { + "type": "string" + }, + "type": "array" + }, + "files": { + "items": { + "$ref": "#/$defs/DpkgFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ] + }, + "DpkgFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "isConfigFile": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "path", + "isConfigFile" + ] + }, + "ELFSecurityFeatures": { + "properties": { + "symbolTableStripped": { + "type": "boolean" + }, + "stackCanary": { + "type": "boolean" + }, + "nx": { + "type": "boolean" + }, + "relRO": { + "type": "string" + }, + "pie": { + "type": "boolean" + }, + "dso": { + "type": "boolean" + }, + "safeStack": { + "type": "boolean" + }, + "cfi": { + "type": "boolean" + }, + "fortify": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "symbolTableStripped", + "nx", + "relRO", + "pie", + "dso" + ] + }, + "ElixirMixLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "ErlangRebarLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "Executable": { + "properties": { + "format": { + "type": "string" + }, + "elfSecurityFeatures": { + "$ref": "#/$defs/ELFSecurityFeatures" + } + }, + "type": "object", + "required": [ + "format" + ] + }, + "File": { + "properties": { + "id": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Coordinates" + }, + "metadata": { + "$ref": "#/$defs/FileMetadataEntry" + }, + "contents": { + "type": "string" + }, + "digests": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + }, + "licenses": { + "items": { + "$ref": "#/$defs/FileLicense" + }, + "type": "array" + }, + "executable": { + "$ref": "#/$defs/Executable" + } + }, + "type": "object", + "required": [ + "id", + "location" + ] + }, + "FileLicense": { + "properties": { + "value": { + "type": "string" + }, + "spdxExpression": { + "type": "string" + }, + "type": { + "type": "string" + }, + "evidence": { + "$ref": "#/$defs/FileLicenseEvidence" + } + }, + "type": "object", + "required": [ + "value", + "spdxExpression", + "type" + ] + }, + "FileLicenseEvidence": { + "properties": { + "confidence": { + "type": "integer" + }, + "offset": { + "type": "integer" + }, + "extent": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "confidence", + "offset", + "extent" + ] + }, + "FileMetadataEntry": { + "properties": { + "mode": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "linkDestination": { + "type": "string" + }, + "userID": { + "type": "integer" + }, + "groupID": { + "type": "integer" + }, + "mimeType": { + "type": "string" + }, + "size": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "mode", + "type", + "userID", + "groupID", + "mimeType", + "size" + ] + }, + "GoModuleBuildinfoEntry": { + "properties": { + "goBuildSettings": { + "$ref": "#/$defs/KeyValues" + }, + "goCompiledVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "h1Digest": { + "type": "string" + }, + "mainModule": { + "type": "string" + }, + "goCryptoSettings": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "goCompiledVersion", + "architecture" + ] + }, + "GoModuleEntry": { + "properties": { + "h1Digest": { + "type": "string" + } + }, + "type": "object" + }, + "HaskellHackageStackEntry": { + "properties": { + "pkgHash": { + "type": "string" + } + }, + "type": "object" + }, + "HaskellHackageStackLockEntry": { + "properties": { + "pkgHash": { + "type": "string" + }, + "snapshotURL": { + "type": "string" + } + }, + "type": "object" + }, + "IDLikes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "JavaArchive": { + "properties": { + "virtualPath": { + "type": "string" + }, + "manifest": { + "$ref": "#/$defs/JavaManifest" + }, + "pomProperties": { + "$ref": "#/$defs/JavaPomProperties" + }, + "pomProject": { + "$ref": "#/$defs/JavaPomProject" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "virtualPath" + ] + }, + "JavaManifest": { + "properties": { + "main": { + "$ref": "#/$defs/KeyValues" + }, + "sections": { + "items": { + "$ref": "#/$defs/KeyValues" + }, + "type": "array" + } + }, + "type": "object" + }, + "JavaPomParent": { + "properties": { + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "type": "object", + "required": [ + "groupId", + "artifactId", + "version" + ] + }, + "JavaPomProject": { + "properties": { + "path": { + "type": "string" + }, + "parent": { + "$ref": "#/$defs/JavaPomParent" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "groupId", + "artifactId", + "version", + "name" + ] + }, + "JavaPomProperties": { + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "extraFields": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path", + "name", + "groupId", + "artifactId", + "version" + ] + }, + "JavascriptNpmPackage": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + }, + "private": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "homepage", + "description", + "url", + "private" + ] + }, + "JavascriptNpmPackageLockEntry": { + "properties": { + "resolved": { + "type": "string" + }, + "integrity": { + "type": "string" + } + }, + "type": "object", + "required": [ + "resolved", + "integrity" + ] + }, + "JavascriptYarnLockEntry": { + "properties": { + "resolved": { + "type": "string" + }, + "integrity": { + "type": "string" + } + }, + "type": "object", + "required": [ + "resolved", + "integrity" + ] + }, + "KeyValue": { + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "key", + "value" + ] + }, + "KeyValues": { + "items": { + "$ref": "#/$defs/KeyValue" + }, + "type": "array" + }, + "License": { + "properties": { + "value": { + "type": "string" + }, + "spdxExpression": { + "type": "string" + }, + "type": { + "type": "string" + }, + "urls": { + "items": { + "type": "string" + }, + "type": "array" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "value", + "spdxExpression", + "type", + "urls", + "locations" + ] + }, + "LinuxKernelArchive": { + "properties": { + "name": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "version": { + "type": "string" + }, + "extendedVersion": { + "type": "string" + }, + "buildTime": { + "type": "string" + }, + "author": { + "type": "string" + }, + "format": { + "type": "string" + }, + "rwRootFS": { + "type": "boolean" + }, + "swapDevice": { + "type": "integer" + }, + "rootDevice": { + "type": "integer" + }, + "videoMode": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "architecture", + "version" + ] + }, + "LinuxKernelModule": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "path": { + "type": "string" + }, + "description": { + "type": "string" + }, + "author": { + "type": "string" + }, + "license": { + "type": "string" + }, + "kernelVersion": { + "type": "string" + }, + "versionMagic": { + "type": "string" + }, + "parameters": { + "patternProperties": { + ".*": { + "$ref": "#/$defs/LinuxKernelModuleParameter" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "LinuxKernelModuleParameter": { + "properties": { + "type": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "type": "object" + }, + "LinuxRelease": { + "properties": { + "prettyName": { + "type": "string" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "idLike": { + "$ref": "#/$defs/IDLikes" + }, + "version": { + "type": "string" + }, + "versionID": { + "type": "string" + }, + "versionCodename": { + "type": "string" + }, + "buildID": { + "type": "string" + }, + "imageID": { + "type": "string" + }, + "imageVersion": { + "type": "string" + }, + "variant": { + "type": "string" + }, + "variantID": { + "type": "string" + }, + "homeURL": { + "type": "string" + }, + "supportURL": { + "type": "string" + }, + "bugReportURL": { + "type": "string" + }, + "privacyPolicyURL": { + "type": "string" + }, + "cpeName": { + "type": "string" + }, + "supportEnd": { + "type": "string" + } + }, + "type": "object" + }, + "Location": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + }, + "accessPath": { + "type": "string" + }, + "annotations": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path", + "accessPath" + ] + }, + "MicrosoftKbPatch": { + "properties": { + "product_id": { + "type": "string" + }, + "kb": { + "type": "string" + } + }, + "type": "object", + "required": [ + "product_id", + "kb" + ] + }, + "NixStoreEntry": { + "properties": { + "outputHash": { + "type": "string" + }, + "output": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "outputHash", + "files" + ] + }, + "Package": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "foundBy": { + "type": "string" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + }, + "licenses": { + "$ref": "#/$defs/licenses" + }, + "language": { + "type": "string" + }, + "cpes": { + "$ref": "#/$defs/cpes" + }, + "purl": { + "type": "string" + }, + "metadataType": { + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/AlpmDbEntry" + }, + { + "$ref": "#/$defs/ApkDbEntry" + }, + { + "$ref": "#/$defs/BinarySignature" + }, + { + "$ref": "#/$defs/CConanFileEntry" + }, + { + "$ref": "#/$defs/CConanInfoEntry" + }, + { + "$ref": "#/$defs/CConanLockEntry" + }, + { + "$ref": "#/$defs/CocoaPodfileLockEntry" + }, + { + "$ref": "#/$defs/DartPubspecLockEntry" + }, + { + "$ref": "#/$defs/DotnetDepsEntry" + }, + { + "$ref": "#/$defs/DotnetPortableExecutableEntry" + }, + { + "$ref": "#/$defs/DpkgDbEntry" + }, + { + "$ref": "#/$defs/ElixirMixLockEntry" + }, + { + "$ref": "#/$defs/ErlangRebarLockEntry" + }, + { + "$ref": "#/$defs/GoModuleBuildinfoEntry" + }, + { + "$ref": "#/$defs/GoModuleEntry" + }, + { + "$ref": "#/$defs/HaskellHackageStackEntry" + }, + { + "$ref": "#/$defs/HaskellHackageStackLockEntry" + }, + { + "$ref": "#/$defs/JavaArchive" + }, + { + "$ref": "#/$defs/JavascriptNpmPackage" + }, + { + "$ref": "#/$defs/JavascriptNpmPackageLockEntry" + }, + { + "$ref": "#/$defs/JavascriptYarnLockEntry" + }, + { + "$ref": "#/$defs/LinuxKernelArchive" + }, + { + "$ref": "#/$defs/LinuxKernelModule" + }, + { + "$ref": "#/$defs/MicrosoftKbPatch" + }, + { + "$ref": "#/$defs/NixStoreEntry" + }, + { + "$ref": "#/$defs/PhpComposerInstalledEntry" + }, + { + "$ref": "#/$defs/PhpComposerLockEntry" + }, + { + "$ref": "#/$defs/PortageDbEntry" + }, + { + "$ref": "#/$defs/PythonPackage" + }, + { + "$ref": "#/$defs/PythonPipRequirementsEntry" + }, + { + "$ref": "#/$defs/PythonPipfileLockEntry" + }, + { + "$ref": "#/$defs/PythonPoetryLockEntry" + }, + { + "$ref": "#/$defs/RDescription" + }, + { + "$ref": "#/$defs/RpmArchive" + }, + { + "$ref": "#/$defs/RpmDbEntry" + }, + { + "$ref": "#/$defs/RubyGemspec" + }, + { + "$ref": "#/$defs/RustCargoAuditEntry" + }, + { + "$ref": "#/$defs/RustCargoLockEntry" + }, + { + "$ref": "#/$defs/SwiftPackageManagerLockEntry" + } + ] + } + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "foundBy", + "locations", + "licenses", + "language", + "cpes", + "purl" + ] + }, + "PhpComposerAuthors": { + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "PhpComposerExternalReference": { + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": "string" + }, + "reference": { + "type": "string" + }, + "shasum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "type", + "url", + "reference" + ] + }, + "PhpComposerInstalledEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "require": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "provide": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "require-dev": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "suggest": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": { + "type": "string" + }, + "notification-url": { + "type": "string" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array" + }, + "description": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array" + }, + "time": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ] + }, + "PhpComposerLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "require": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "provide": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "require-dev": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "suggest": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": { + "type": "string" + }, + "notification-url": { + "type": "string" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array" + }, + "description": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array" + }, + "time": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ] + }, + "PortageDbEntry": { + "properties": { + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/PortageFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "installedSize", + "files" + ] + }, + "PortageFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PythonDirectURLOriginInfo": { + "properties": { + "url": { + "type": "string" + }, + "commitId": { + "type": "string" + }, + "vcs": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "PythonFileDigest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "PythonFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/PythonFileDigest" + }, + "size": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PythonPackage": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorEmail": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/PythonFileRecord" + }, + "type": "array" + }, + "sitePackagesRootPath": { + "type": "string" + }, + "topLevelPackages": { + "items": { + "type": "string" + }, + "type": "array" + }, + "directUrlOrigin": { + "$ref": "#/$defs/PythonDirectURLOriginInfo" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "authorEmail", + "platform", + "sitePackagesRootPath" + ] + }, + "PythonPipRequirementsEntry": { + "properties": { + "name": { + "type": "string" + }, + "extras": { + "items": { + "type": "string" + }, + "type": "array" + }, + "versionConstraint": { + "type": "string" + }, + "url": { + "type": "string" + }, + "markers": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "versionConstraint" + ] + }, + "PythonPipfileLockEntry": { + "properties": { + "hashes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "index": { + "type": "string" + } + }, + "type": "object", + "required": [ + "hashes", + "index" + ] + }, + "PythonPoetryLockEntry": { + "properties": { + "index": { + "type": "string" + } + }, + "type": "object", + "required": [ + "index" + ] + }, + "RDescription": { + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "author": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "url": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repository": { + "type": "string" + }, + "built": { + "type": "string" + }, + "needsCompilation": { + "type": "boolean" + }, + "imports": { + "items": { + "type": "string" + }, + "type": "array" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array" + }, + "suggests": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "Relationship": { + "properties": { + "parent": { + "type": "string" + }, + "child": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": true + }, + "type": "object", + "required": [ + "parent", + "child", + "type" + ] + }, + "RpmArchive": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "vendor": { + "type": "string" + }, + "modularityLabel": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/RpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "vendor", + "files" + ] + }, + "RpmDbEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "vendor": { + "type": "string" + }, + "modularityLabel": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/RpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "vendor", + "files" + ] + }, + "RpmFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "mode": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "userName": { + "type": "string" + }, + "groupName": { + "type": "string" + }, + "flags": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "mode", + "size", + "digest", + "userName", + "groupName", + "flags" + ] + }, + "RubyGemspec": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "RustCargoAuditEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source" + ] + }, + "RustCargoLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string" + }, + "checksum": { + "type": "string" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "checksum", + "dependencies" + ] + }, + "Schema": { + "properties": { + "version": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "version", + "url" + ] + }, + "Source": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": true + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "metadata" + ] + }, + "SwiftPackageManagerLockEntry": { + "properties": { + "revision": { + "type": "string" + } + }, + "type": "object", + "required": [ + "revision" + ] + }, + "cpes": { + "items": { + "$ref": "#/$defs/CPE" + }, + "type": "array" + }, + "licenses": { + "items": { + "$ref": "#/$defs/License" + }, + "type": "array" + } + } +} diff --git a/syft/internal/packagemetadata/generated.go b/syft/internal/packagemetadata/generated.go index d2e119c226e..ddee53a5b6c 100644 --- a/syft/internal/packagemetadata/generated.go +++ b/syft/internal/packagemetadata/generated.go @@ -36,6 +36,7 @@ func AllTypes() []any { pkg.PortageEntry{}, pkg.PythonPackage{}, pkg.PythonPipfileLockEntry{}, + pkg.PythonPoetryLockEntry{}, pkg.PythonRequirementsEntry{}, pkg.RDescription{}, pkg.RpmArchive{}, @@ -44,5 +45,6 @@ func AllTypes() []any { pkg.RustBinaryAuditEntry{}, pkg.RustCargoLockEntry{}, pkg.SwiftPackageManagerResolvedEntry{}, + pkg.YarnLockEntry{}, } } diff --git a/syft/internal/packagemetadata/names.go b/syft/internal/packagemetadata/names.go index 2bec9866937..c997069dc1c 100644 --- a/syft/internal/packagemetadata/names.go +++ b/syft/internal/packagemetadata/names.go @@ -86,11 +86,13 @@ var jsonTypes = makeJSONTypes( jsonNames(pkg.NixStoreEntry{}, "nix-store-entry", "NixStoreMetadata"), jsonNames(pkg.NpmPackage{}, "javascript-npm-package", "NpmPackageJsonMetadata"), jsonNames(pkg.NpmPackageLockEntry{}, "javascript-npm-package-lock-entry", "NpmPackageLockJsonMetadata"), + jsonNames(pkg.YarnLockEntry{}, "javascript-yarn-lock-entry", "YarnLockJsonMetadata"), jsonNames(pkg.PhpComposerLockEntry{}, "php-composer-lock-entry", "PhpComposerJsonMetadata"), jsonNamesWithoutLookup(pkg.PhpComposerInstalledEntry{}, "php-composer-installed-entry", "PhpComposerJsonMetadata"), // the legacy value is split into two types, where the other is preferred jsonNames(pkg.PortageEntry{}, "portage-db-entry", "PortageMetadata"), jsonNames(pkg.PythonPackage{}, "python-package", "PythonPackageMetadata"), jsonNames(pkg.PythonPipfileLockEntry{}, "python-pipfile-lock-entry", "PythonPipfileLockMetadata"), + jsonNames(pkg.PythonPoetryLockEntry{}, "python-poetry-lock-entry", "PythonPoetryLockMetadata"), jsonNames(pkg.PythonRequirementsEntry{}, "python-pip-requirements-entry", "PythonRequirementsMetadata"), jsonNames(pkg.ErlangRebarLockEntry{}, "erlang-rebar-lock-entry", "RebarLockMetadataType"), jsonNames(pkg.RDescription{}, "r-description", "RDescriptionFileMetadataType"), diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 990f5b01021..893de6cc0bf 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -110,7 +110,7 @@ func newPnpmPackage(resolver file.Resolver, location file.Location, name, versio ) } -func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string) pkg.Package { +func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string, resolved string, integrity string) pkg.Package { var licenseSet pkg.LicenseSet if cfg.SearchRemoteLicenses { @@ -123,7 +123,6 @@ func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location fi log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err) } } - return finalizeLockPkg( resolver, location, @@ -135,6 +134,7 @@ func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location fi PURL: packageURL(name, version), Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{Resolved: resolved, Integrity: integrity}, }, ) } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index c9a243be90d..73dcd30018d 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -34,11 +34,18 @@ var ( // `resolved "https://registry.yarnpkg.com/@4lolo/resize-observer-polyfill/-/resize-observer-polyfill-1.5.2.tgz#58868fc7224506236b5550d0c68357f0a874b84b"` // would return "@4lolo/resize-observer-polyfill" and "1.5.2" packageURLExp = regexp.MustCompile(`^\s+resolved\s+"https://registry\.(?:yarnpkg\.com|npmjs\.org)/(.+?)/-/(?:.+?)-(\d+\..+?)\.tgz`) -) -const ( - noPackage = "" - noVersion = "" + // resolvedExp matches the resolved of the dependency in yarn.lock + // For example: + // resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" + // would return "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" + resolvedExp = regexp.MustCompile(`^\s+resolved\s+"(.+?)"`) + + // integrityExp matches the integrity of the dependency in yarn.lock + // For example: + // integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== + // would return "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA=="" + integrityExp = regexp.MustCompile(`^\s+integrity\s+([^\s]+)`) ) type genericYarnLockAdapter struct { @@ -59,37 +66,43 @@ func (a genericYarnLockAdapter) parseYarnLock(_ context.Context, resolver file.R } var pkgs []pkg.Package + var currentPackage, currentVersion, currentResolved, currentIntegrity string + scanner := bufio.NewScanner(reader) parsedPackages := strset.New() - currentPackage := noPackage - currentVersion := noVersion for scanner.Scan() { line := scanner.Text() - if packageName := findPackageName(line); packageName != noPackage { + if packageName := findPackageName(line); packageName != "" { // When we find a new package, check if we have unsaved identifiers - if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion)) + if currentPackage != "" && currentVersion != "" && !parsedPackages.Has(currentPackage+"@"+currentVersion) { + pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion, currentResolved, currentIntegrity)) parsedPackages.Add(currentPackage + "@" + currentVersion) } currentPackage = packageName - } else if version := findPackageVersion(line); version != noVersion { + } else if version := findPackageVersion(line); version != "" { + currentVersion = version + } else if packageName, version, resolved := findResolvedPackageAndVersion(line); packageName != "" && version != "" && resolved != "" { + currentResolved = resolved + currentPackage = packageName currentVersion = version - } else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Has(packageName+"@"+version) { - pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, packageName, version)) - parsedPackages.Add(packageName + "@" + version) + } else if integrity := findIntegrity(line); integrity != "" && !parsedPackages.Has(currentPackage+"@"+currentVersion) { + pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion, currentResolved, integrity)) + parsedPackages.Add(currentPackage + "@" + currentVersion) // Cleanup to indicate no unsaved identifiers - currentPackage = noPackage - currentVersion = noVersion + currentPackage = "" + currentVersion = "" + currentResolved = "" + currentIntegrity = "" } } // check if we have valid unsaved data after end-of-file has reached - if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion)) + if currentPackage != "" && currentVersion != "" && !parsedPackages.Has(currentPackage+"@"+currentVersion) { + pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion, currentResolved, currentIntegrity)) parsedPackages.Add(currentPackage + "@" + currentVersion) } @@ -107,7 +120,7 @@ func findPackageName(line string) string { return matches[1] } - return noPackage + return "" } func findPackageVersion(line string) string { @@ -115,13 +128,25 @@ func findPackageVersion(line string) string { return matches[1] } - return noVersion + return "" } -func findPackageAndVersion(line string) (string, string) { +func findResolvedPackageAndVersion(line string) (string, string, string) { + var resolved string + if matches := resolvedExp.FindStringSubmatch(line); len(matches) >= 2 { + resolved = matches[1] + } if matches := packageURLExp.FindStringSubmatch(line); len(matches) >= 2 { - return matches[1], matches[2] + return matches[1], matches[2], resolved + } + + return "", "", "" +} + +func findIntegrity(line string) string { + if matches := integrityExp.FindStringSubmatch(line); len(matches) >= 2 { + return matches[1] } - return noPackage, noVersion + return "" } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index 8a0bdee6d9c..20fac12fe6c 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -28,6 +28,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/%40babel/code-frame@7.10.4", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "@types/minimatch", @@ -36,6 +37,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/%40types/minimatch@3.0.3", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "@types/qs", @@ -44,6 +46,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/%40types/qs@6.9.4", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "ajv", @@ -52,6 +55,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/ajv@6.12.3", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "asn1.js", @@ -60,6 +64,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/asn1.js@4.10.1", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "atob", @@ -68,6 +73,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/atob@2.1.2", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "aws-sdk", @@ -76,6 +82,7 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "c0n-fab_u.laTION", @@ -84,6 +91,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/c0n-fab_u.laTION@7.7.7", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, { Name: "jhipster-core", @@ -92,6 +100,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/jhipster-core@7.3.4", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{}, }, } @@ -112,6 +121,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/%40babel/code-frame@7.10.4", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a", + Integrity: "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + }, }, { Name: "@types/minimatch", @@ -120,6 +133,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/%40types/minimatch@3.0.3", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d", + Integrity: "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + }, }, { Name: "@types/qs", @@ -128,6 +145,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/%40types/qs@6.9.4", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.4.tgz#a59e851c1ba16c0513ea123830dd639a0a15cb6a", + Integrity: "sha512-+wYo+L6ZF6BMoEjtf8zB2esQsqdV6WsjRK/GP9WOgLPrq87PbNWgIxS76dS5uvl/QXtHGakZmwTznIfcPXcKlQ==", + }, }, { Name: "ajv", @@ -136,6 +157,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/ajv@6.12.3", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/ajv/-/ajv-6.12.3.tgz#18c5af38a111ddeb4f2697bd78d68abc1cabd706", + Integrity: "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==", + }, }, { Name: "asn1.js", @@ -144,6 +169,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/asn1.js@4.10.1", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/asn1.js/-/asn1.js-4.10.1.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0", + Integrity: "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==", + }, }, { Name: "atob", @@ -153,6 +182,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/atob@2.1.2", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9", + Integrity: "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + }, }, { Name: "aws-sdk", @@ -161,6 +194,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/aws-sdk@2.706.0", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953", + Integrity: "sha512-7GT+yrB5Wb/zOReRdv/Pzkb2Qt+hz6B/8FGMVaoysX3NryHvQUdz7EQWi5yhg9CxOjKxdw5lFwYSs69YlSp1KA==", + }, }, { Name: "jhipster-core", @@ -169,8 +206,11 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/jhipster-core@7.3.4", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/jhipster-core/-/jhipster-core-7.3.4.tgz#c34b8c97c7f4e8b7518dae015517e2112c73cc80", + Integrity: "sha512-AUhT69kNkqppaJZVfan/xnKG4Gs9Ggj7YLtTZFVe+xg+THrbMb5Ng7PL07PDlDw4KAEA33GMCwuAf65E8EpC4g==", + }, }, - { Name: "something-i-made-up", Version: "7.7.7", @@ -178,6 +218,10 @@ func TestParseYarnLock(t *testing.T) { PURL: "pkg:npm/something-i-made-up@7.7.7", Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/something-i-made-up/-/c0n-fab_u.laTION-7.7.7.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0", + Integrity: "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==", + }, }, } @@ -221,6 +265,10 @@ func TestSearchYarnForLicenses(t *testing.T) { Licenses: pkg.NewLicenseSet(pkg.NewLicense("MIT")), Language: pkg.JavaScript, Type: pkg.NpmPkg, + Metadata: pkg.YarnLockEntry{ + Resolved: "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a", + Integrity: "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + }, }, }, }, diff --git a/syft/pkg/cataloger/python/package.go b/syft/pkg/cataloger/python/package.go index 7e6f39a22fa..f2d51ac3225 100644 --- a/syft/pkg/cataloger/python/package.go +++ b/syft/pkg/cataloger/python/package.go @@ -25,7 +25,7 @@ func newPackageForIndex(name, version string, locations ...file.Location) pkg.Pa return p } -func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPipfileLockEntry, locations ...file.Location) pkg.Package { +func newPackageForIndexWithMetadata(name, version string, metadata interface{}, locations ...file.Location) pkg.Package { p := pkg.Package{ Name: name, Version: version, diff --git a/syft/pkg/cataloger/python/parse_pipfile_lock.go b/syft/pkg/cataloger/python/parse_pipfile_lock.go index 7e1181f4aa0..b2100060a5d 100644 --- a/syft/pkg/cataloger/python/parse_pipfile_lock.go +++ b/syft/pkg/cataloger/python/parse_pipfile_lock.go @@ -62,7 +62,7 @@ func parsePipfileLock(_ context.Context, _ file.Resolver, _ *generic.Environment if pkgMeta.Index != "" { index = sourcesMap[pkgMeta.Index] } else { - // https://pipenv.pypa.io/en/latest/advanced/#specifying-package-indexes + // https://pipenv.pypa.io/en/latest/indexes.html index = "https://pypi.org/simple" } version := strings.TrimPrefix(pkgMeta.Version, "==") diff --git a/syft/pkg/cataloger/python/parse_poetry_lock.go b/syft/pkg/cataloger/python/parse_poetry_lock.go index dfcab6a10e1..dd7660efd21 100644 --- a/syft/pkg/cataloger/python/parse_poetry_lock.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock.go @@ -15,13 +15,18 @@ import ( // integrity check var _ generic.Parser = parsePoetryLock -type poetryMetadata struct { +type poetryPackageSource struct { + URL string `toml:"url"` +} + +type poetryPackages struct { Packages []struct { - Name string `toml:"name"` - Version string `toml:"version"` - Category string `toml:"category"` - Description string `toml:"description"` - Optional bool `toml:"optional"` + Name string `toml:"name"` + Version string `toml:"version"` + Category string `toml:"category"` + Description string `toml:"description"` + Optional bool `toml:"optional"` + Source poetryPackageSource `toml:"source"` } `toml:"package"` } @@ -32,7 +37,7 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment, return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %w", err) } - metadata := poetryMetadata{} + metadata := poetryPackages{} err = tree.Unmarshal(&metadata) if err != nil { return nil, nil, fmt.Errorf("unable to parse poetry.lock: %w", err) @@ -40,11 +45,19 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment, var pkgs []pkg.Package for _, p := range metadata.Packages { + var index string + if p.Source.URL != "" { + index = p.Source.URL + } else { + // https://python-poetry.org/docs/repositories/ + index = "https://pypi.org/simple" + } pkgs = append( pkgs, - newPackageForIndex( + newPackageForIndexWithMetadata( p.Name, p.Version, + pkg.PythonPoetryLockEntry{Index: index}, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), ) diff --git a/syft/pkg/cataloger/python/parse_poetry_lock_test.go b/syft/pkg/cataloger/python/parse_poetry_lock_test.go index fd6d1bdc805..6cc20504f84 100644 --- a/syft/pkg/cataloger/python/parse_poetry_lock_test.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock_test.go @@ -20,6 +20,7 @@ func TestParsePoetryLock(t *testing.T) { Locations: locations, Language: pkg.Python, Type: pkg.PythonPkg, + Metadata: pkg.PythonPoetryLockEntry{Index: "https://test.pypi.org/simple"}, }, { Name: "alabaster", @@ -28,6 +29,7 @@ func TestParsePoetryLock(t *testing.T) { Locations: locations, Language: pkg.Python, Type: pkg.PythonPkg, + Metadata: pkg.PythonPoetryLockEntry{Index: "https://pypi.org/simple"}, }, { Name: "appnope", @@ -36,6 +38,7 @@ func TestParsePoetryLock(t *testing.T) { Locations: locations, Language: pkg.Python, Type: pkg.PythonPkg, + Metadata: pkg.PythonPoetryLockEntry{Index: "https://pypi.org/simple"}, }, { Name: "asciitree", @@ -44,6 +47,7 @@ func TestParsePoetryLock(t *testing.T) { Locations: locations, Language: pkg.Python, Type: pkg.PythonPkg, + Metadata: pkg.PythonPoetryLockEntry{Index: "https://pypi.org/simple"}, }, } diff --git a/syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock b/syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock index 489f1df6d6d..6d20352ff08 100644 --- a/syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock +++ b/syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock @@ -12,6 +12,11 @@ natsort = "*" six = "*" sphinx = "*" +[package.source] +type = "legacy" +url = "https://test.pypi.org/simple" +reference = "test" + [package.extras] deploy = ["bumpversion", "twine", "wheel"] docs = ["sphinx", "sphinx-rtd-theme"] diff --git a/syft/pkg/npm.go b/syft/pkg/npm.go index 4abeace174b..c39539bd66b 100644 --- a/syft/pkg/npm.go +++ b/syft/pkg/npm.go @@ -16,3 +16,9 @@ type NpmPackageLockEntry struct { Resolved string `mapstructure:"resolved" json:"resolved"` Integrity string `mapstructure:"integrity" json:"integrity"` } + +// YarnLockEntry represents a single entry section of a yarn.lock file. +type YarnLockEntry struct { + Resolved string `mapstructure:"resolved" json:"resolved"` + Integrity string `mapstructure:"integrity" json:"integrity"` +} diff --git a/syft/pkg/python.go b/syft/pkg/python.go index 1976ea7c806..477d6fb23c2 100644 --- a/syft/pkg/python.go +++ b/syft/pkg/python.go @@ -61,6 +61,11 @@ type PythonPipfileLockEntry struct { Index string `mapstructure:"index" json:"index"` } +// PythonPoetryLockEntry represents a single package entry within a Pipfile.lock file. +type PythonPoetryLockEntry struct { + Index string `mapstructure:"index" json:"index"` +} + // PythonRequirementsEntry represents a single entry within a [*-]requirements.txt file. type PythonRequirementsEntry struct { Name string `json:"name" mapstruct:"Name"`