diff --git a/fixtures/configs-extra-dbs/db/osv-1.json b/fixtures/configs-extra-dbs/db/osv-1.json new file mode 100644 index 00000000..c18862f0 --- /dev/null +++ b/fixtures/configs-extra-dbs/db/osv-1.json @@ -0,0 +1,17 @@ +{ + "id": "OSV-1", + "affected": [ + { + "package": { + "ecosystem": "npm", + "name": "request" + } + }, + { + "package": { + "ecosystem": "npm", + "name": "@cypress/request" + } + } + ] +} diff --git a/fixtures/configs-extra-dbs/db/osv-2.json b/fixtures/configs-extra-dbs/db/osv-2.json new file mode 100644 index 00000000..bd60877b --- /dev/null +++ b/fixtures/configs-extra-dbs/db/osv-2.json @@ -0,0 +1,3 @@ +{ + "id": "OSV-2" +} diff --git a/fixtures/configs-extra-dbs/db/osv-3.json b/fixtures/configs-extra-dbs/db/osv-3.json new file mode 100644 index 00000000..0bd8467e --- /dev/null +++ b/fixtures/configs-extra-dbs/db/osv-3.json @@ -0,0 +1,3 @@ +{ + "id": "OSV-3" +} diff --git a/main.go b/main.go index 7ed8ed56..d8537797 100644 --- a/main.go +++ b/main.go @@ -170,7 +170,7 @@ func describeDB(db database.DB) string { color.YellowString("%d", tt.BatchSize), ) case *database.ZipDB: - count := len(tt.Vulnerabilities(true)) + count := tt.VulnerabilitiesCount return fmt.Sprintf( "%s %s, including withdrawn - last updated %s", @@ -179,7 +179,7 @@ func describeDB(db database.DB) string { tt.UpdatedAt, ) case *database.DirDB: - count := len(tt.Vulnerabilities(true)) + count := tt.VulnerabilitiesCount return fmt.Sprintf( "%s %s, including withdrawn", diff --git a/main_test.go b/main_test.go index 2e6ae22e..d0494a2d 100644 --- a/main_test.go +++ b/main_test.go @@ -1036,12 +1036,12 @@ func TestRun_Configs(t *testing.T) { wantStdout: ` Loaded the following OSV databases: api#https://example.com/v1 (using batches of 1000) - dir#file:/fixtures/configs-extra-dbs (0 vulnerabilities, including withdrawn) + dir#file:/fixtures/configs-extra-dbs (3 vulnerabilities, including withdrawn) zip#https://example.com/osvs/all fixtures/configs-extra-dbs/yarn.lock: found 0 packages Using config at fixtures/configs-extra-dbs/.osv-detector.yaml (0 ignores) Using db api#https://example.com/v1 (using batches of 1000) - Using db dir#file:/fixtures/configs-extra-dbs (0 vulnerabilities, including withdrawn) + Using db dir#file:/fixtures/configs-extra-dbs (3 vulnerabilities, including withdrawn) no known vulnerabilities found `, diff --git a/pkg/database/dir.go b/pkg/database/dir.go index 2c21a475..3ef8ec79 100644 --- a/pkg/database/dir.go +++ b/pkg/database/dir.go @@ -29,7 +29,7 @@ var ErrDirPathWrongProtocol = errors.New("directory path must start with \"file: // load walks the filesystem starting with the working directory within the local path, // loading all OSVs found along the way. func (db *DirDB) load() error { - db.vulnerabilities = []OSV{} + db.vulnerabilities = make(map[string][]OSV) if !strings.HasPrefix(db.LocalPath, "file:") { return ErrDirPathWrongProtocol @@ -78,7 +78,7 @@ func (db *DirDB) load() error { return nil } - db.vulnerabilities = append(db.vulnerabilities, pa) + db.addVulnerability(pa) return nil }) diff --git a/pkg/database/dir_test.go b/pkg/database/dir_test.go index fd61fb88..fa55838c 100644 --- a/pkg/database/dir_test.go +++ b/pkg/database/dir_test.go @@ -9,7 +9,17 @@ import ( func TestNewDirDB(t *testing.T) { t.Parallel() - osvs := []database.OSV{{ID: "OSV-1"}, {ID: "OSV-2"}, {ID: "GHSA-1234"}} + osvs := []database.OSV{ + withDefaultAffected("OSV-1"), + withDefaultAffected("OSV-2"), + { + ID: "GHSA-1234", + Affected: []database.Affected{ + {Package: database.Package{Ecosystem: "npm", Name: "request"}}, + {Package: database.Package{Ecosystem: "npm", Name: "@cypress/request"}}, + }, + }, + } db, err := database.NewDirDB(database.Config{URL: "file:/fixtures/db"}, false) @@ -69,7 +79,7 @@ func TestNewDirDB_DoesNotExist(t *testing.T) { func TestNewDirDB_WorkingDirectory(t *testing.T) { t.Parallel() - osvs := []database.OSV{{ID: "OSV-1"}} + osvs := []database.OSV{withDefaultAffected("OSV-1")} db, err := database.NewDirDB(database.Config{URL: "file:/fixtures/db", WorkingDirectory: "nested-1"}, false) diff --git a/pkg/database/fixtures/db/file.json b/pkg/database/fixtures/db/file.json index 0ba8200f..46aaf413 100644 --- a/pkg/database/fixtures/db/file.json +++ b/pkg/database/fixtures/db/file.json @@ -1,3 +1,17 @@ { - "id": "GHSA-1234" + "id": "GHSA-1234", + "affected": [ + { + "package": { + "ecosystem": "npm", + "name": "request" + } + }, + { + "package": { + "ecosystem": "npm", + "name": "@cypress/request" + } + } + ] } diff --git a/pkg/database/fixtures/db/nested-1/osv-1.json b/pkg/database/fixtures/db/nested-1/osv-1.json index 7c531474..c56ca9da 100644 --- a/pkg/database/fixtures/db/nested-1/osv-1.json +++ b/pkg/database/fixtures/db/nested-1/osv-1.json @@ -1,3 +1,12 @@ { - "id": "OSV-1" + "id": "OSV-1", + "affected": [ + { + "package": { + "name": "mine", + "ecosystem": "PyPi" + }, + "versions": [] + } + ] } diff --git a/pkg/database/fixtures/db/nested-2/osv-2.json b/pkg/database/fixtures/db/nested-2/osv-2.json index bd60877b..8018c2b1 100644 --- a/pkg/database/fixtures/db/nested-2/osv-2.json +++ b/pkg/database/fixtures/db/nested-2/osv-2.json @@ -1,3 +1,12 @@ { - "id": "OSV-2" + "id": "OSV-2", + "affected": [ + { + "package": { + "name": "mine", + "ecosystem": "PyPi" + }, + "versions": [] + } + ] } diff --git a/pkg/database/mem-check.go b/pkg/database/mem-check.go index 96949679..c0c35f5d 100644 --- a/pkg/database/mem-check.go +++ b/pkg/database/mem-check.go @@ -7,19 +7,39 @@ import ( // an OSV database that lives in-memory, and can be used by other structs // that handle loading the vulnerabilities from where ever type memDB struct { - vulnerabilities []OSV + vulnerabilities map[string][]OSV + VulnerabilitiesCount int } -func (db *memDB) Vulnerabilities(includeWithdrawn bool) []OSV { - if includeWithdrawn { - return db.vulnerabilities +func (db *memDB) addVulnerability(osv OSV) { + db.VulnerabilitiesCount++ + + for _, affected := range osv.Affected { + hash := string(affected.Package.Ecosystem) + "-" + affected.Package.NormalizedName() + vulns := db.vulnerabilities[hash] + + if vulns == nil { + vulns = []OSV{} + } + + db.vulnerabilities[hash] = append(vulns, osv) } +} +func (db *memDB) Vulnerabilities(includeWithdrawn bool) []OSV { var vulnerabilities []OSV + ids := make(map[string]struct{}) - for _, vulnerability := range db.vulnerabilities { - if vulnerability.Withdrawn == nil { - vulnerabilities = append(vulnerabilities, vulnerability) + for _, vulns := range db.vulnerabilities { + for _, vulnerability := range vulns { + if _, ok := ids[vulnerability.ID]; ok { + continue + } + + if (vulnerability.Withdrawn == nil) || includeWithdrawn { + vulnerabilities = append(vulnerabilities, vulnerability) + ids[vulnerability.ID] = struct{}{} + } } } @@ -29,9 +49,13 @@ func (db *memDB) Vulnerabilities(includeWithdrawn bool) []OSV { func (db *memDB) VulnerabilitiesAffectingPackage(pkg internal.PackageDetails) Vulnerabilities { var vulnerabilities Vulnerabilities - for _, vulnerability := range db.Vulnerabilities(false) { - if vulnerability.IsAffected(pkg) && !vulnerabilities.Includes(vulnerability) { - vulnerabilities = append(vulnerabilities, vulnerability) + hash := string(pkg.Ecosystem) + "-" + pkg.Name + + if vulns, ok := db.vulnerabilities[hash]; ok { + for _, vulnerability := range vulns { + if vulnerability.Withdrawn == nil && vulnerability.IsAffected(pkg) && !vulnerabilities.Includes(vulnerability) { + vulnerabilities = append(vulnerabilities, vulnerability) + } } } diff --git a/pkg/database/zip.go b/pkg/database/zip.go index ddf89b7a..b2319cc6 100644 --- a/pkg/database/zip.go +++ b/pkg/database/zip.go @@ -152,7 +152,7 @@ func (db *ZipDB) loadZipFile(zipFile *zip.File) { return } - db.vulnerabilities = append(db.vulnerabilities, osv) + db.addVulnerability(osv) } // load fetches a zip archive of the OSV database and loads known vulnerabilities @@ -162,7 +162,7 @@ func (db *ZipDB) loadZipFile(zipFile *zip.File) { // so that a new version of the archive is only downloaded if it has been // modified, per HTTP caching standards. func (db *ZipDB) load() error { - db.vulnerabilities = []OSV{} + db.vulnerabilities = make(map[string][]OSV) body, err := db.fetchZip() diff --git a/pkg/database/zip_test.go b/pkg/database/zip_test.go index 07b0de94..30d37999 100644 --- a/pkg/database/zip_test.go +++ b/pkg/database/zip_test.go @@ -18,6 +18,21 @@ import ( "testing" ) +func withDefaultAffected(id string) database.OSV { + return database.OSV{ + ID: id, + Affected: []database.Affected{ + { + Package: database.Package{ + Name: "mine", + Ecosystem: "PyPi", + }, + Versions: database.Versions{}, + }, + }, + } +} + func expectDBToHaveOSVs( t *testing.T, db interface { @@ -137,11 +152,11 @@ func TestNewZippedDB_Offline_WithCache(t *testing.T) { date := "Fri, 17 Jun 2022 22:28:13 GMT" osvs := []database.OSV{ - {ID: "GHSA-1"}, - {ID: "GHSA-2"}, - {ID: "GHSA-3"}, - {ID: "GHSA-4"}, - {ID: "GHSA-5"}, + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), } ts, cleanup := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { @@ -154,11 +169,11 @@ func TestNewZippedDB_Offline_WithCache(t *testing.T) { ETag: "", Date: date, Body: zipOSVs(t, map[string]database.OSV{ - "GHSA-1.json": {ID: "GHSA-1"}, - "GHSA-2.json": {ID: "GHSA-2"}, - "GHSA-3.json": {ID: "GHSA-3"}, - "GHSA-4.json": {ID: "GHSA-4"}, - "GHSA-5.json": {ID: "GHSA-5"}, + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), }), }) @@ -204,20 +219,20 @@ func TestNewZippedDB_Online_WithoutCache(t *testing.T) { t.Parallel() osvs := []database.OSV{ - {ID: "GHSA-1"}, - {ID: "GHSA-2"}, - {ID: "GHSA-3"}, - {ID: "GHSA-4"}, - {ID: "GHSA-5"}, + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), } ts, cleanup := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(zipOSVs(t, map[string]database.OSV{ - "GHSA-1.json": {ID: "GHSA-1"}, - "GHSA-2.json": {ID: "GHSA-2"}, - "GHSA-3.json": {ID: "GHSA-3"}, - "GHSA-4.json": {ID: "GHSA-4"}, - "GHSA-5.json": {ID: "GHSA-5"}, + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), })) }) defer cleanup() @@ -254,9 +269,9 @@ func TestNewZippedDB_Online_WithCache(t *testing.T) { date := "Fri, 18 Jun 2022 22:28:13 GMT" osvs := []database.OSV{ - {ID: "GHSA-1"}, - {ID: "GHSA-2"}, - {ID: "GHSA-3"}, + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), } ts, cleanup := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { @@ -273,9 +288,9 @@ func TestNewZippedDB_Online_WithCache(t *testing.T) { ETag: "", Date: date, Body: zipOSVs(t, map[string]database.OSV{ - "GHSA-1.json": {ID: "GHSA-1"}, - "GHSA-2.json": {ID: "GHSA-2"}, - "GHSA-3.json": {ID: "GHSA-3"}, + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), }), }) @@ -297,11 +312,11 @@ func TestNewZippedDB_Online_WithOldCache(t *testing.T) { date := "Fri, 17 Jun 2022 22:28:13 GMT" osvs := []database.OSV{ - {ID: "GHSA-1"}, - {ID: "GHSA-2"}, - {ID: "GHSA-3"}, - {ID: "GHSA-4"}, - {ID: "GHSA-5"}, + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), } ts, cleanup := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { @@ -311,11 +326,11 @@ func TestNewZippedDB_Online_WithOldCache(t *testing.T) { w.Header().Set("Date", "Today") _, _ = w.Write(zipOSVs(t, map[string]database.OSV{ - "GHSA-1.json": {ID: "GHSA-1"}, - "GHSA-2.json": {ID: "GHSA-2"}, - "GHSA-3.json": {ID: "GHSA-3"}, - "GHSA-4.json": {ID: "GHSA-4"}, - "GHSA-5.json": {ID: "GHSA-5"}, + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), })) }) defer cleanup() @@ -325,9 +340,9 @@ func TestNewZippedDB_Online_WithOldCache(t *testing.T) { ETag: "", Date: date, Body: zipOSVs(t, map[string]database.OSV{ - "GHSA-1.json": {ID: "GHSA-1"}, - "GHSA-2.json": {ID: "GHSA-2"}, - "GHSA-3.json": {ID: "GHSA-3"}, + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), }), }) @@ -348,16 +363,16 @@ func TestNewZippedDB_Online_WithBadCache(t *testing.T) { t.Parallel() osvs := []database.OSV{ - {ID: "GHSA-1"}, - {ID: "GHSA-2"}, - {ID: "GHSA-3"}, + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), } ts, cleanup := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(zipOSVs(t, map[string]database.OSV{ - "GHSA-1.json": {ID: "GHSA-1"}, - "GHSA-2.json": {ID: "GHSA-2"}, - "GHSA-3.json": {ID: "GHSA-3"}, + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), })) }) defer cleanup() @@ -376,15 +391,15 @@ func TestNewZippedDB_Online_WithBadCache(t *testing.T) { func TestNewZippedDB_FileChecks(t *testing.T) { t.Parallel() - osvs := []database.OSV{{ID: "GHSA-1234"}, {ID: "GHSA-4321"}} + osvs := []database.OSV{withDefaultAffected("GHSA-1234"), withDefaultAffected("GHSA-4321")} ts, cleanup := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(zipOSVs(t, map[string]database.OSV{ - "file.json": {ID: "GHSA-1234"}, + "file.json": withDefaultAffected("GHSA-1234"), // only files with .json suffix should be loaded - "file.yaml": {ID: "GHSA-5678"}, + "file.yaml": withDefaultAffected("GHSA-5678"), // (no longer) special case for the GH security database - "advisory-database-main/advisories/unreviewed/file.json": {ID: "GHSA-4321"}, + "advisory-database-main/advisories/unreviewed/file.json": withDefaultAffected("GHSA-4321"), })) }) defer cleanup() @@ -401,13 +416,13 @@ func TestNewZippedDB_FileChecks(t *testing.T) { func TestNewZippedDB_WorkingDirectory(t *testing.T) { t.Parallel() - osvs := []database.OSV{{ID: "GHSA-1234"}, {ID: "GHSA-5678"}} + osvs := []database.OSV{withDefaultAffected("GHSA-1234"), withDefaultAffected("GHSA-5678")} ts, cleanup := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(zipOSVs(t, map[string]database.OSV{ - "reviewed/file.json": {ID: "GHSA-1234"}, - "reviewed/nested/file.json": {ID: "GHSA-5678"}, - "unreviewed/file.json": {ID: "GHSA-4321"}, + "reviewed/file.json": withDefaultAffected("GHSA-1234"), + "reviewed/nested/file.json": withDefaultAffected("GHSA-5678"), + "unreviewed/file.json": withDefaultAffected("GHSA-4321"), })) }) defer cleanup()