From d7b9cc70b075b01ec8d4c550bdfde1eacc0a0d68 Mon Sep 17 00:00:00 2001 From: Laurent Goderre Date: Fri, 2 Feb 2024 13:40:18 -0500 Subject: [PATCH] Add Erlang OTP Application cataloger (#2403) * Add cataloger for Erlang OTP applications Signed-off-by: Laurent Goderre * Add OTP Package type and Purl for ErLang Signed-off-by: Laurent Goderre * remove erlang OTP metadata type Signed-off-by: Alex Goodman * use OTP purl type Signed-off-by: Alex Goodman * restore otp fixture and adjust tests for dir-only results Signed-off-by: Alex Goodman --------- Signed-off-by: Laurent Goderre Signed-off-by: Alex Goodman Co-authored-by: Alex Goodman --- .../catalog_packages_cases_test.go | 8 ++++ .../test/integration/catalog_packages_test.go | 3 +- .../image-pkg-coverage/pkgs/erlang/accept.app | 10 ++++ go.mod | 2 +- go.sum | 4 +- internal/task/package_tasks.go | 1 + .../internal/spdxutil/helpers/source_info.go | 2 + .../spdxutil/helpers/source_info_test.go | 8 ++++ syft/pkg/cataloger/erlang/cataloger.go | 7 ++- syft/pkg/cataloger/erlang/cataloger_test.go | 27 ++++++++++- syft/pkg/cataloger/erlang/package.go | 34 +++++++++++-- syft/pkg/cataloger/erlang/parse_otp_app.go | 48 +++++++++++++++++++ .../cataloger/erlang/parse_otp_app_test.go | 43 +++++++++++++++++ syft/pkg/cataloger/erlang/parse_rebar_lock.go | 2 +- .../test-fixtures/glob-paths/src/rabbitmq.app | 1 + .../erlang/test-fixtures/rabbitmq.app | 18 +++++++ syft/pkg/language.go | 4 +- syft/pkg/type.go | 11 ++++- syft/pkg/type_test.go | 4 ++ 19 files changed, 223 insertions(+), 14 deletions(-) create mode 100644 cmd/syft/internal/test/integration/test-fixtures/image-pkg-coverage/pkgs/erlang/accept.app create mode 100644 syft/pkg/cataloger/erlang/parse_otp_app.go create mode 100644 syft/pkg/cataloger/erlang/parse_otp_app_test.go create mode 100644 syft/pkg/cataloger/erlang/test-fixtures/glob-paths/src/rabbitmq.app create mode 100644 syft/pkg/cataloger/erlang/test-fixtures/rabbitmq.app diff --git a/cmd/syft/internal/test/integration/catalog_packages_cases_test.go b/cmd/syft/internal/test/integration/catalog_packages_cases_test.go index 3df13864a00..c3542213b8b 100644 --- a/cmd/syft/internal/test/integration/catalog_packages_cases_test.go +++ b/cmd/syft/internal/test/integration/catalog_packages_cases_test.go @@ -368,6 +368,14 @@ var dirOnlyTestCases = []testCase{ "unicode_util_compat": "0.7.0", }, }, + { + name: "find ErLang OTP applications", + pkgType: pkg.ErlangOTPPkg, + pkgLanguage: pkg.Erlang, + pkgInfo: map[string]string{ + "accept": "0.3.5", + }, + }, { name: "find swift package manager packages", pkgType: pkg.SwiftPkg, diff --git a/cmd/syft/internal/test/integration/catalog_packages_test.go b/cmd/syft/internal/test/integration/catalog_packages_test.go index c0144e987da..010eae751ee 100644 --- a/cmd/syft/internal/test/integration/catalog_packages_test.go +++ b/cmd/syft/internal/test/integration/catalog_packages_test.go @@ -57,8 +57,8 @@ func TestPkgCoverageImage(t *testing.T) { definedLanguages.Remove(pkg.Swift.String()) definedLanguages.Remove(pkg.CPP.String()) definedLanguages.Remove(pkg.Haskell.String()) - definedLanguages.Remove(pkg.Erlang.String()) definedLanguages.Remove(pkg.Elixir.String()) + definedLanguages.Remove(pkg.Erlang.String()) observedPkgs := strset.New() definedPkgs := strset.New() @@ -71,6 +71,7 @@ func TestPkgCoverageImage(t *testing.T) { definedPkgs.Remove(string(pkg.GoModulePkg)) definedPkgs.Remove(string(pkg.RustPkg)) definedPkgs.Remove(string(pkg.DartPubPkg)) + definedPkgs.Remove(string(pkg.ErlangOTPPkg)) definedPkgs.Remove(string(pkg.CocoapodsPkg)) definedPkgs.Remove(string(pkg.ConanPkg)) definedPkgs.Remove(string(pkg.HackagePkg)) diff --git a/cmd/syft/internal/test/integration/test-fixtures/image-pkg-coverage/pkgs/erlang/accept.app b/cmd/syft/internal/test/integration/test-fixtures/image-pkg-coverage/pkgs/erlang/accept.app new file mode 100644 index 00000000000..a2b960a7684 --- /dev/null +++ b/cmd/syft/internal/test/integration/test-fixtures/image-pkg-coverage/pkgs/erlang/accept.app @@ -0,0 +1,10 @@ +{application,accept, + [{description,"Accept header(s) for Erlang/Elixir"}, + {vsn,"0.3.5"}, + {registered,[]}, + {applications,[kernel,stdlib]}, + {env,[]}, + {modules, ['accept_encoding_header','accept_header','accept_neg','accept_parser']}, + {maintainers,["Ilya Khaprov"]}, + {licenses,["MIT"]}, + {links,[{"Github","https://github.com/deadtrickster/accept"}]}]}. diff --git a/go.mod b/go.mod index 7ef6368ee55..ed5f1d05b2b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b - github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 + github.com/anchore/packageurl-go v0.1.1-0.20240202171727-877e1747d426 github.com/anchore/stereoscope v0.0.2-0.20240201224129-37291e81936d github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // we are hinting brotli to latest due to warning when installing archiver v3: diff --git a/go.sum b/go.sum index bf581fe0022..d3b93ea1de2 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 h1:AV7qjwMcM4r8wFhJq3jLRztew3ywIyPTRapl2T1s9o8= -github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4= +github.com/anchore/packageurl-go v0.1.1-0.20240202171727-877e1747d426 h1:agoiZchSf1Nnnos1azwIg5hk5Ao9TzZNBD9++AChGEg= +github.com/anchore/packageurl-go v0.1.1-0.20240202171727-877e1747d426/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4= github.com/anchore/stereoscope v0.0.2-0.20240201224129-37291e81936d h1:v+kf6J76l5nWvdvxptgyLXWr45G8CGVScL4AAISi3nI= github.com/anchore/stereoscope v0.0.2-0.20240201224129-37291e81936d/go.mod h1:uydT2ful8TY7Hr1WH1V1ZecSq/2TqXpAsGkMiy7lxD0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= diff --git a/internal/task/package_tasks.go b/internal/task/package_tasks.go index f5c17e6f3cc..c3adeb64896 100644 --- a/internal/task/package_tasks.go +++ b/internal/task/package_tasks.go @@ -57,6 +57,7 @@ func DefaultPackageTaskFactories() PackageTaskFactories { newSimplePackageTaskFactory(dotnet.NewDotnetDepsCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dotnet", "c#"), newSimplePackageTaskFactory(elixir.NewMixLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "elixir"), newSimplePackageTaskFactory(erlang.NewRebarLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "erlang"), + newSimplePackageTaskFactory(erlang.NewOTPCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "erlang", "otp"), newSimplePackageTaskFactory(haskell.NewHackageCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "haskell", "hackage", "cabal"), newPackageTaskFactory( func(cfg CatalogingFactoryConfig) pkg.Cataloger { diff --git a/syft/format/internal/spdxutil/helpers/source_info.go b/syft/format/internal/spdxutil/helpers/source_info.go index 04af05eb35b..693d3c7b97f 100644 --- a/syft/format/internal/spdxutil/helpers/source_info.go +++ b/syft/format/internal/spdxutil/helpers/source_info.go @@ -46,6 +46,8 @@ func SourceInfo(p pkg.Package) string { answer = "acquired package info from cabal or stack manifest files" case pkg.HexPkg: answer = "acquired package info from rebar3 or mix manifest file" + case pkg.ErlangOTPPkg: + answer = "acquired package info from ErLang application resource file" case pkg.LinuxKernelPkg: answer = "acquired package info from linux kernel archive" case pkg.LinuxKernelModulePkg: diff --git a/syft/format/internal/spdxutil/helpers/source_info_test.go b/syft/format/internal/spdxutil/helpers/source_info_test.go index 98f87791654..5993cb07b8f 100644 --- a/syft/format/internal/spdxutil/helpers/source_info_test.go +++ b/syft/format/internal/spdxutil/helpers/source_info_test.go @@ -199,6 +199,14 @@ func Test_SourceInfo(t *testing.T) { "from rebar3 or mix manifest file", }, }, + { + input: pkg.Package{ + Type: pkg.ErlangOTPPkg, + }, + expected: []string{ + "from ErLang application resource file", + }, + }, { input: pkg.Package{ Type: pkg.LinuxKernelPkg, diff --git a/syft/pkg/cataloger/erlang/cataloger.go b/syft/pkg/cataloger/erlang/cataloger.go index ed0ebc923a6..5b8be9b31b5 100644 --- a/syft/pkg/cataloger/erlang/cataloger.go +++ b/syft/pkg/cataloger/erlang/cataloger.go @@ -1,5 +1,5 @@ /* -Package erlang provides a concrete Cataloger implementation relating to packages within the Erlang language ecosystem. +Package erlang provides concrete Catalogers implementation relating to packages within the Erlang language ecosystem. */ package erlang @@ -13,3 +13,8 @@ func NewRebarLockCataloger() pkg.Cataloger { return generic.NewCataloger("erlang-rebar-lock-cataloger"). WithParserByGlobs(parseRebarLock, "**/rebar.lock") } + +func NewOTPCataloger() pkg.Cataloger { + return generic.NewCataloger("erlang-otp-application-cataloger"). + WithParserByGlobs(parseOTPApp, "**/*.app") +} diff --git a/syft/pkg/cataloger/erlang/cataloger_test.go b/syft/pkg/cataloger/erlang/cataloger_test.go index 1d959b10389..d29ce3e14dc 100644 --- a/syft/pkg/cataloger/erlang/cataloger_test.go +++ b/syft/pkg/cataloger/erlang/cataloger_test.go @@ -6,7 +6,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) -func TestCataloger_Globs(t *testing.T) { +func TestCatalogerRebar_Globs(t *testing.T) { tests := []struct { name string fixture string @@ -30,3 +30,28 @@ func TestCataloger_Globs(t *testing.T) { }) } } + +func TestCatalogerOTP_Globs(t *testing.T) { + tests := []struct { + name string + fixture string + expected []string + }{ + { + name: "obtain OTP resource files", + fixture: "test-fixtures/glob-paths", + expected: []string{ + "src/rabbitmq.app", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pkgtest.NewCatalogTester(). + FromDirectory(t, test.fixture). + ExpectsResolverContentQueries(test.expected). + TestCataloger(t, NewOTPCataloger()) + }) + } +} diff --git a/syft/pkg/cataloger/erlang/package.go b/syft/pkg/cataloger/erlang/package.go index c9009876bd7..60157996ce4 100644 --- a/syft/pkg/cataloger/erlang/package.go +++ b/syft/pkg/cataloger/erlang/package.go @@ -6,13 +6,13 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func newPackage(d pkg.ErlangRebarLockEntry, locations ...file.Location) pkg.Package { +func newPackageFromRebar(d pkg.ErlangRebarLockEntry, locations ...file.Location) pkg.Package { p := pkg.Package{ Name: d.Name, Version: d.Version, Language: pkg.Erlang, Locations: file.NewLocationSet(locations...), - PURL: packageURL(d), + PURL: packageURLFromRebar(d), Type: pkg.HexPkg, Metadata: d, } @@ -22,7 +22,7 @@ func newPackage(d pkg.ErlangRebarLockEntry, locations ...file.Location) pkg.Pack return p } -func packageURL(m pkg.ErlangRebarLockEntry) string { +func packageURLFromRebar(m pkg.ErlangRebarLockEntry) string { var qualifiers packageurl.Qualifiers return packageurl.NewPackageURL( @@ -34,3 +34,31 @@ func packageURL(m pkg.ErlangRebarLockEntry) string { "", ).ToString() } + +func newPackageFromOTP(name, version string, locations ...file.Location) pkg.Package { + p := pkg.Package{ + Name: name, + Version: version, + Language: pkg.Erlang, + Locations: file.NewLocationSet(locations...), + PURL: packageURLFromOTP(name, version), + Type: pkg.ErlangOTPPkg, + } + + p.SetID() + + return p +} + +func packageURLFromOTP(name, version string) string { + var qualifiers packageurl.Qualifiers + + return packageurl.NewPackageURL( + packageurl.TypeOTP, + "", + name, + version, + qualifiers, + "", + ).ToString() +} diff --git a/syft/pkg/cataloger/erlang/parse_otp_app.go b/syft/pkg/cataloger/erlang/parse_otp_app.go new file mode 100644 index 00000000000..dcea41c18e1 --- /dev/null +++ b/syft/pkg/cataloger/erlang/parse_otp_app.go @@ -0,0 +1,48 @@ +package erlang + +import ( + "context" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +// parseOTPApp parses a OTP *.app files to a package objects +func parseOTPApp(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + doc, err := parseErlang(reader) + if err != nil { + // there are multiple file formats that use the *.app extension, so it's possible that this is not an OTP app file at all + // ... which means we should not return an error here + log.WithFields("error", err).Trace("unable to parse Erlang OTP app") + return nil, nil, nil + } + + var packages []pkg.Package + + root := doc.Get(0) + + name := root.Get(1).String() + + keys := root.Get(2) + + for _, key := range keys.Slice() { + if key.Get(0).String() == "vsn" { + version := key.Get(1).String() + + p := newPackageFromOTP( + name, version, + reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), + ) + + packages = append(packages, p) + } + } + + return packages, nil, nil +} + +// integrity check +var _ generic.Parser = parseOTPApp diff --git a/syft/pkg/cataloger/erlang/parse_otp_app_test.go b/syft/pkg/cataloger/erlang/parse_otp_app_test.go new file mode 100644 index 00000000000..9c0abf16edb --- /dev/null +++ b/syft/pkg/cataloger/erlang/parse_otp_app_test.go @@ -0,0 +1,43 @@ +package erlang + +import ( + "testing" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" +) + +func TestParseOTPApplication(t *testing.T) { + tests := []struct { + fixture string + expected []pkg.Package + }{ + { + fixture: "test-fixtures/rabbitmq.app", + expected: []pkg.Package{ + { + Name: "rabbit", + Version: "3.12.10", + Language: pkg.Erlang, + Type: pkg.ErlangOTPPkg, + PURL: "pkg:otp/rabbit@3.12.10", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + // TODO: relationships are not under test + var expectedRelationships []artifact.Relationship + + for idx := range test.expected { + test.expected[idx].Locations = file.NewLocationSet(file.NewLocation(test.fixture)) + } + + pkgtest.TestFileParser(t, test.fixture, parseOTPApp, test.expected, expectedRelationships) + }) + } +} diff --git a/syft/pkg/cataloger/erlang/parse_rebar_lock.go b/syft/pkg/cataloger/erlang/parse_rebar_lock.go index e69f101072b..a173ec81bfd 100644 --- a/syft/pkg/cataloger/erlang/parse_rebar_lock.go +++ b/syft/pkg/cataloger/erlang/parse_rebar_lock.go @@ -50,7 +50,7 @@ func parseRebarLock(_ context.Context, _ file.Resolver, _ *generic.Environment, version = versionNode.Get(2).Get(1).String() } - p := newPackage( + p := newPackageFromRebar( pkg.ErlangRebarLockEntry{ Name: name, Version: version, diff --git a/syft/pkg/cataloger/erlang/test-fixtures/glob-paths/src/rabbitmq.app b/syft/pkg/cataloger/erlang/test-fixtures/glob-paths/src/rabbitmq.app new file mode 100644 index 00000000000..ceae2148587 --- /dev/null +++ b/syft/pkg/cataloger/erlang/test-fixtures/glob-paths/src/rabbitmq.app @@ -0,0 +1 @@ +bogus erlang file \ No newline at end of file diff --git a/syft/pkg/cataloger/erlang/test-fixtures/rabbitmq.app b/syft/pkg/cataloger/erlang/test-fixtures/rabbitmq.app new file mode 100644 index 00000000000..a1b58992c22 --- /dev/null +++ b/syft/pkg/cataloger/erlang/test-fixtures/rabbitmq.app @@ -0,0 +1,18 @@ +{application, 'rabbit', [ + {description, "RabbitMQ"}, + {vsn, "3.12.10"}, + {id, "v3.12.9-9-g1f61ca8"}, + {modules, ['amqqueue','background_gc']}, + {optional_applications, []}, + {env, [ + {memory_monitor_interval, 2500}, + {disk_free_limit, 50000000}, %% 50MB + {msg_store_index_module, rabbit_msg_store_ets_index}, + {backing_queue_module, rabbit_variable_queue}, + %% 0 ("no limit") would make a better default, but that + %% breaks the QPid Java client + {frame_max, 131072}, + %% see rabbitmq-server#1593 + {channel_max, 2047} + ]} +]}. \ No newline at end of file diff --git a/syft/pkg/language.go b/syft/pkg/language.go index 889d31070e0..347c225083d 100644 --- a/syft/pkg/language.go +++ b/syft/pkg/language.go @@ -80,7 +80,7 @@ func LanguageByName(name string) Language { return Rust case packageurl.TypePub, string(DartPubPkg), string(Dart): return Dart - case packageurl.TypeDotnet, packageurl.TypeNuget: + case string(Dotnet), ".net", packageurl.TypeNuget: return Dotnet case packageurl.TypeCocoapods, packageurl.TypeSwift, string(CocoapodsPkg), string(SwiftPkg): return Swift @@ -88,7 +88,7 @@ func LanguageByName(name string) Language { return CPP case packageurl.TypeHackage, string(Haskell): return Haskell - case packageurl.TypeHex, "beam", "elixir", "erlang": + case packageurl.TypeHex, packageurl.TypeOTP, "beam", "elixir", "erlang": // should we support returning multiple languages to support this case? // answer: no. We want this to definitively answer "which language does this package represent?" // which might not be possible in all cases. See for more context: https://github.com/package-url/purl-spec/pull/178 diff --git a/syft/pkg/type.go b/syft/pkg/type.go index 788e584a5bf..7d836127c3d 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -18,6 +18,7 @@ const ( DartPubPkg Type = "dart-pub" DebPkg Type = "deb" DotnetPkg Type = "dotnet" + ErlangOTPPkg Type = "erlang-otp" GemPkg Type = "gem" GithubActionPkg Type = "github-action" GithubActionWorkflowPkg Type = "github-action-workflow" @@ -51,6 +52,7 @@ var AllPkgs = []Type{ DartPubPkg, DebPkg, DotnetPkg, + ErlangOTPPkg, GemPkg, GithubActionPkg, GithubActionWorkflowPkg, @@ -91,7 +93,9 @@ func (t Type) PackageURLType() string { case DebPkg: return "deb" case DotnetPkg: - return packageurl.TypeDotnet + return "dotnet" + case ErlangOTPPkg: + return packageurl.TypeOTP case GemPkg: return packageurl.TypeGem case HexPkg: @@ -146,6 +150,7 @@ func TypeFromPURL(p string) Type { return TypeByName(ptype) } +//nolint:funlen func TypeByName(name string) Type { switch name { case packageurl.TypeDebian: @@ -172,7 +177,7 @@ func TypeByName(name string) Type { return RustPkg case packageurl.TypePub: return DartPubPkg - case packageurl.TypeDotnet: + case "dotnet": // here to support legacy use cases return DotnetPkg case packageurl.TypeCocoapods: return CocoapodsPkg @@ -184,6 +189,8 @@ func TypeByName(name string) Type { return PortagePkg case packageurl.TypeHex: return HexPkg + case packageurl.TypeOTP: + return ErlangOTPPkg case "linux-kernel": return LinuxKernelPkg case "linux-kernel-module": diff --git a/syft/pkg/type_test.go b/syft/pkg/type_test.go index 64d8e87c35a..b2fa458a214 100644 --- a/syft/pkg/type_test.go +++ b/syft/pkg/type_test.go @@ -83,6 +83,10 @@ func TestTypeFromPURL(t *testing.T) { purl: "pkg:hex/hpax/hpax@0.1.1", expected: HexPkg, }, + { + purl: "pkg:otp/accept@0.3.5", + expected: ErlangOTPPkg, + }, { purl: "pkg:generic/linux-kernel@5.10.15", expected: LinuxKernelPkg,