From 210d13e174c90fc17263fb1f6cc2c2a366265ebd Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Mon, 19 Feb 2024 14:53:18 +0100 Subject: [PATCH 1/3] feat: add support for java Use sonatype API to query for updates in maven central repo and publish those into feeds --- pkg/config/scheduledfeed.go | 7 ++ pkg/feeds/feed.go | 2 +- pkg/feeds/maven/README.md | 13 ++++ pkg/feeds/maven/maven.go | 135 ++++++++++++++++++++++++++++++++++ pkg/feeds/maven/maven_test.go | 114 ++++++++++++++++++++++++++++ 5 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 pkg/feeds/maven/README.md create mode 100644 pkg/feeds/maven/maven.go create mode 100644 pkg/feeds/maven/maven_test.go diff --git a/pkg/config/scheduledfeed.go b/pkg/config/scheduledfeed.go index d9453d6d..5e54e726 100644 --- a/pkg/config/scheduledfeed.go +++ b/pkg/config/scheduledfeed.go @@ -17,6 +17,7 @@ import ( "github.com/ossf/package-feeds/pkg/feeds" "github.com/ossf/package-feeds/pkg/feeds/crates" "github.com/ossf/package-feeds/pkg/feeds/goproxy" + "github.com/ossf/package-feeds/pkg/feeds/maven" "github.com/ossf/package-feeds/pkg/feeds/npm" "github.com/ossf/package-feeds/pkg/feeds/nuget" "github.com/ossf/package-feeds/pkg/feeds/packagist" @@ -179,6 +180,8 @@ func (fc FeedConfig) ToFeed(eventHandler *events.Handler) (feeds.ScheduledFeed, return npm.New(fc.Options, eventHandler) case nuget.FeedName: return nuget.New(fc.Options) + case maven.FeedName: + return maven.New(fc.Options) case pypi.FeedName: return pypi.New(fc.Options, eventHandler) case packagist.FeedName: @@ -222,6 +225,10 @@ func Default() *ScheduledFeedConfig { Type: nuget.FeedName, Options: defaultFeedOptions, }, + { + Type: maven.FeedName, + Options: defaultFeedOptions, + }, { Type: packagist.FeedName, Options: defaultFeedOptions, diff --git a/pkg/feeds/feed.go b/pkg/feeds/feed.go index 690d59e4..be1869fc 100644 --- a/pkg/feeds/feed.go +++ b/pkg/feeds/feed.go @@ -70,7 +70,7 @@ func NewArtifact(created time.Time, name, version, artifactID, feed string) *Pac func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package { filteredPackages := []*Package{} for _, pkg := range pkgs { - if pkg.CreatedDate.After(cutoff) { + if pkg.CreatedDate.UTC().After(cutoff) { filteredPackages = append(filteredPackages, pkg) } } diff --git a/pkg/feeds/maven/README.md b/pkg/feeds/maven/README.md new file mode 100644 index 00000000..bc81b87d --- /dev/null +++ b/pkg/feeds/maven/README.md @@ -0,0 +1,13 @@ +# maven Feed + +This feed allows polling of package updates from central.sonatype, polling Maven central repository. + +## Configuration options + +The `packages` field is not supported by the maven feed. + + +``` +feeds: +- type: maven +``` \ No newline at end of file diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go new file mode 100644 index 00000000..71c76761 --- /dev/null +++ b/pkg/feeds/maven/maven.go @@ -0,0 +1,135 @@ +package maven + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" +) + +const ( + FeedName = "maven" + indexPath = "/api/internal/browse/components" +) + +type Feed struct { + baseURL string + options feeds.FeedOptions +} + +func New(feedOptions feeds.FeedOptions) (*Feed, error) { + if feedOptions.Packages != nil { + return nil, feeds.UnsupportedOptionError{ + Feed: FeedName, + Option: "packages", + } + } + return &Feed{ + baseURL: "https://central.sonatype.com/" + indexPath, + options: feedOptions, + }, nil +} + +// Package represents package information +type LatestVersionInfo struct { + Version string `json:"version"` + TimestampUnixWithMS int64 `json:"timestampUnixWithMS"` +} + +type Package struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + LatestVersionInfo LatestVersionInfo `json:"latestVersionInfo"` +} + +// Response represents the response structure from Sonatype API +type Response struct { + Components []Package `json:"components"` +} + +// fetchPackages fetches packages from Sonatype API for the given page +func (feed Feed) fetchPackages(page int) ([]Package, error) { + // Define the request payload + payload := map[string]interface{}{ + "page": page, + "size": 20, + "sortField": "publishedDate", + "sortDirection": "desc", + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("error encoding JSON: %v", err) + } + + // Send POST request to Sonatype API + resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return nil, fmt.Errorf("error sending request: %v", err) + } + defer resp.Body.Close() + + // Decode response + var response Response + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, fmt.Errorf("error decoding response: %v", err) + } + return response.Components, nil +} + +func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) { + pkgs := []*feeds.Package{} + var errs []error + + page := 0 + for { + // Fetch packages from Sonatype API for the current page + packages, err := feed.fetchPackages(page) + if err != nil { + errs = append(errs, err) + break + } + + // Iterate over packages + hasToCut := false + for _, pkg := range packages { + // convert published to date to compare with cutoff + if pkg.LatestVersionInfo.TimestampUnixWithMS > cutoff.UnixMilli() { + // Append package to pkgs + timestamp := time.Unix(pkg.LatestVersionInfo.TimestampUnixWithMS/1000, 0) + packageName := pkg.Namespace + ":" + pkg.Name + + newPkg := feeds.NewPackage(timestamp, packageName, pkg.LatestVersionInfo.Version, FeedName) + pkgs = append(pkgs, newPkg) + } else { + // Break the loop if the cutoff date is reached + hasToCut = true + } + } + + // Move to the next page + page++ + + // Check if the loop should be terminated + if len(pkgs) == 0 || hasToCut { + break + } + } + + newCutoff := feeds.FindCutoff(cutoff, pkgs) + pkgs = feeds.ApplyCutoff(pkgs, cutoff) + + return pkgs, newCutoff, errs +} + +func (feed Feed) GetName() string { + return FeedName +} + +func (feed Feed) GetFeedOptions() feeds.FeedOptions { + return feed.options +} diff --git a/pkg/feeds/maven/maven_test.go b/pkg/feeds/maven/maven_test.go new file mode 100644 index 00000000..a656cd99 --- /dev/null +++ b/pkg/feeds/maven/maven_test.go @@ -0,0 +1,114 @@ +package maven + +import ( + "net/http" + "testing" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" + testutils "github.com/ossf/package-feeds/pkg/utils/test" +) + +func TestMavenLatest(t *testing.T) { + t.Parallel() + + handlers := map[string]testutils.HTTPHandlerFunc{ + indexPath: mavenPackageResponse, + } + srv := testutils.HTTPServerMock(handlers) + + feed, err := New(feeds.FeedOptions{}) + if err != nil { + t.Fatalf("Failed to create Maven feed: %v", err) + } + feed.baseURL = srv.URL + "/api/internal/browse/components" + + cutoff := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC) + pkgs, gotCutoff, errs := feed.Latest(cutoff) + + if len(errs) != 0 { + t.Fatalf("feed.Latest returned error: %v", err) + } + + // Returned cutoff should match the newest package creation time of packages retrieved. + wantCutoff := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + if gotCutoff.UTC().Sub(wantCutoff).Abs() > time.Second { + t.Errorf("Latest() cutoff %v, want %v", gotCutoff, wantCutoff) + } + if pkgs[0].Name != "com.github.example:project" { + t.Errorf("Unexpected package `%s` found in place of expected `com.github.example:project`", pkgs[0].Name) + } + if pkgs[0].Version != "1.0.0" { + t.Errorf("Unexpected version `%s` found in place of expected `1.0.0`", pkgs[0].Version) + } + + for _, p := range pkgs { + if p.Type != FeedName { + t.Errorf("Feed type not set correctly in goproxy package following Latest()") + } + } + +} + +func TestMavenNotFound(t *testing.T) { + t.Parallel() + + handlers := map[string]testutils.HTTPHandlerFunc{ + indexPath: testutils.NotFoundHandlerFunc, + } + srv := testutils.HTTPServerMock(handlers) + + feed, err := New(feeds.FeedOptions{}) + if err != nil { + t.Fatalf("Failed to create Maven feed: %v", err) + } + feed.baseURL = srv.URL + "/api/internal/browse/components" + + cutoff := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) + + _, gotCutoff, errs := feed.Latest(cutoff) + if cutoff != gotCutoff { + t.Error("feed.Latest() cutoff should be unchanged if an error is returned") + } + if len(errs) == 0 { + t.Fatalf("feed.Latest() was successful when an error was expected") + } +} + +func mavenPackageResponse(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + responseJSON := ` + { + "components": [ + { + "id": "pkg:maven/com.github.example/project", + "type": "COMPONENT", + "namespace": "com.github.example", + "name": "project", + "version": "1.0.0", + "publishedEpochMillis": 946684800000, + "latestVersionInfo": { + "version": "1.0.0", + "timestampUnixWithMS": 946684800000 + } + }, + { + "id": "pkg:maven/com.github.example/project1", + "type": "COMPONENT", + "namespace": "com.github.example", + "name": "project", + "version": "1.0.0", + "publishedEpochMillis": null, + "latestVersionInfo": { + "version": "1.0.0", + "timestampUnixWithMS": 0 + } + } + ] + } + ` + _, err := w.Write([]byte(responseJSON)) + if err != nil { + http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError) + } +} From eb5ba889531ebc482b04934fe7bcc17cfdd561ff Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Mon, 19 Feb 2024 14:58:06 +0100 Subject: [PATCH 2/3] fix lint --- pkg/feeds/maven/maven.go | 18 +++++++++--------- pkg/feeds/maven/maven_test.go | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go index 71c76761..de3409b1 100644 --- a/pkg/feeds/maven/maven.go +++ b/pkg/feeds/maven/maven.go @@ -33,7 +33,7 @@ func New(feedOptions feeds.FeedOptions) (*Feed, error) { }, nil } -// Package represents package information +// Package represents package information. type LatestVersionInfo struct { Version string `json:"version"` TimestampUnixWithMS int64 `json:"timestampUnixWithMS"` @@ -45,12 +45,12 @@ type Package struct { LatestVersionInfo LatestVersionInfo `json:"latestVersionInfo"` } -// Response represents the response structure from Sonatype API +// Response represents the response structure from Sonatype API. type Response struct { Components []Package `json:"components"` } -// fetchPackages fetches packages from Sonatype API for the given page +// fetchPackages fetches packages from Sonatype API for the given page. func (feed Feed) fetchPackages(page int) ([]Package, error) { // Define the request payload payload := map[string]interface{}{ @@ -62,21 +62,21 @@ func (feed Feed) fetchPackages(page int) ([]Package, error) { jsonPayload, err := json.Marshal(payload) if err != nil { - return nil, fmt.Errorf("error encoding JSON: %v", err) + return nil, fmt.Errorf("error encoding JSON: %w", err) } - // Send POST request to Sonatype API + // Send POST request to Sonatype API. resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload)) if err != nil { - return nil, fmt.Errorf("error sending request: %v", err) + return nil, fmt.Errorf("error sending request: %w", err) } defer resp.Body.Close() - // Decode response + // Decode response. var response Response err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { - return nil, fmt.Errorf("error decoding response: %v", err) + return nil, fmt.Errorf("error decoding response: %w", err) } return response.Components, nil } @@ -87,7 +87,7 @@ func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) page := 0 for { - // Fetch packages from Sonatype API for the current page + // Fetch packages from Sonatype API for the current page. packages, err := feed.fetchPackages(page) if err != nil { errs = append(errs, err) diff --git a/pkg/feeds/maven/maven_test.go b/pkg/feeds/maven/maven_test.go index a656cd99..5fdee128 100644 --- a/pkg/feeds/maven/maven_test.go +++ b/pkg/feeds/maven/maven_test.go @@ -47,7 +47,6 @@ func TestMavenLatest(t *testing.T) { t.Errorf("Feed type not set correctly in goproxy package following Latest()") } } - } func TestMavenNotFound(t *testing.T) { From 1ed83e8fce2c4f8d986691326e5dbd70b8996056 Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Mon, 19 Feb 2024 15:15:24 +0100 Subject: [PATCH 3/3] add error handling --- pkg/feeds/maven/maven.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go index de3409b1..0f1026d9 100644 --- a/pkg/feeds/maven/maven.go +++ b/pkg/feeds/maven/maven.go @@ -72,6 +72,12 @@ func (feed Feed) fetchPackages(page int) ([]Package, error) { } defer resp.Body.Close() + // Handle rate limiting (HTTP status code 429). + if resp.StatusCode == http.StatusTooManyRequests { + time.Sleep(5 * time.Second) + return feed.fetchPackages(page) // Retry the request + } + // Decode response. var response Response err = json.NewDecoder(resp.Body).Decode(&response)