Merge pull request #7 from stacklok/add-maven

feat: add support for java

pkg/config/scheduledfeed.go

@@ -17,6 +17,7 @@ import (
 	"github.com/ossf/package-feeds/pkg/feeds"
 	"github.com/ossf/package-feeds/pkg/feeds/crates"
 	"github.com/ossf/package-feeds/pkg/feeds/goproxy"
+	"github.com/ossf/package-feeds/pkg/feeds/maven"
 	"github.com/ossf/package-feeds/pkg/feeds/npm"
 	"github.com/ossf/package-feeds/pkg/feeds/nuget"
 	"github.com/ossf/package-feeds/pkg/feeds/packagist"
@@ -179,6 +180,8 @@ func (fc FeedConfig) ToFeed(eventHandler *events.Handler) (feeds.ScheduledFeed,
 		return npm.New(fc.Options, eventHandler)
 	case nuget.FeedName:
 		return nuget.New(fc.Options)
+	case maven.FeedName:
+		return maven.New(fc.Options)
 	case pypi.FeedName:
 		return pypi.New(fc.Options, eventHandler)
 	case packagist.FeedName:
@@ -222,6 +225,10 @@ func Default() *ScheduledFeedConfig {
 				Type:    nuget.FeedName,
 				Options: defaultFeedOptions,
 			},
+			{
+				Type:    maven.FeedName,
+				Options: defaultFeedOptions,
+			},
 			{
 				Type:    packagist.FeedName,
 				Options: defaultFeedOptions,

pkg/feeds/feed.go

@@ -70,7 +70,7 @@ func NewArtifact(created time.Time, name, version, artifactID, feed string) *Pac
 func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package {
 	filteredPackages := []*Package{}
 	for _, pkg := range pkgs {
-		if pkg.CreatedDate.After(cutoff) {
+		if pkg.CreatedDate.UTC().After(cutoff) {
 			filteredPackages = append(filteredPackages, pkg)
 		}
 	}

pkg/feeds/maven/README.md

@@ -0,0 +1,13 @@
+# maven Feed
+
+This feed allows polling of package updates from central.sonatype, polling Maven central repository.
+
+## Configuration options
+
+The `packages` field is not supported by the maven feed.
+
+
+```
+feeds:
+- type: maven
+```

pkg/feeds/maven/maven.go

@@ -0,0 +1,141 @@
+package maven
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/ossf/package-feeds/pkg/feeds"
+)
+
+const (
+	FeedName  = "maven"
+	indexPath = "/api/internal/browse/components"
+)
+
+type Feed struct {
+	baseURL string
+	options feeds.FeedOptions
+}
+
+func New(feedOptions feeds.FeedOptions) (*Feed, error) {
+	if feedOptions.Packages != nil {
+		return nil, feeds.UnsupportedOptionError{
+			Feed:   FeedName,
+			Option: "packages",
+		}
+	}
+	return &Feed{
+		baseURL: "https://central.sonatype.com/" + indexPath,
+		options: feedOptions,
+	}, nil
+}
+
+// Package represents package information.
+type LatestVersionInfo struct {
+	Version             string `json:"version"`
+	TimestampUnixWithMS int64  `json:"timestampUnixWithMS"`
+}
+
+type Package struct {
+	Name              string            `json:"name"`
+	Namespace         string            `json:"namespace"`
+	LatestVersionInfo LatestVersionInfo `json:"latestVersionInfo"`
+}
+
+// Response represents the response structure from Sonatype API.
+type Response struct {
+	Components []Package `json:"components"`
+}
+
+// fetchPackages fetches packages from Sonatype API for the given page.
+func (feed Feed) fetchPackages(page int) ([]Package, error) {
+	// Define the request payload
+	payload := map[string]interface{}{
+		"page":          page,
+		"size":          20,
+		"sortField":     "publishedDate",
+		"sortDirection": "desc",
+	}
+
+	jsonPayload, err := json.Marshal(payload)
+	if err != nil {
+		return nil, fmt.Errorf("error encoding JSON: %w", err)
+	}
+
+	// Send POST request to Sonatype API.
+	resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload))
+	if err != nil {
+		return nil, fmt.Errorf("error sending request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Handle rate limiting (HTTP status code 429).
+	if resp.StatusCode == http.StatusTooManyRequests {
+		time.Sleep(5 * time.Second)
+		return feed.fetchPackages(page) // Retry the request
+	}
+
+	// Decode response.
+	var response Response
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding response: %w", err)
+	}
+	return response.Components, nil
+}
+
+func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) {
+	pkgs := []*feeds.Package{}
+	var errs []error
+
+	page := 0
+	for {
+		// Fetch packages from Sonatype API for the current page.
+		packages, err := feed.fetchPackages(page)
+		if err != nil {
+			errs = append(errs, err)
+			break
+		}
+
+		// Iterate over packages
+		hasToCut := false
+		for _, pkg := range packages {
+			// convert published to date to compare with cutoff
+			if pkg.LatestVersionInfo.TimestampUnixWithMS > cutoff.UnixMilli() {
+				// Append package to pkgs
+				timestamp := time.Unix(pkg.LatestVersionInfo.TimestampUnixWithMS/1000, 0)
+				packageName := pkg.Namespace + ":" + pkg.Name
+
+				newPkg := feeds.NewPackage(timestamp, packageName, pkg.LatestVersionInfo.Version, FeedName)
+				pkgs = append(pkgs, newPkg)
+			} else {
+				// Break the loop if the cutoff date is reached
+				hasToCut = true
+			}
+		}
+
+		// Move to the next page
+		page++
+
+		// Check if the loop should be terminated
+		if len(pkgs) == 0 || hasToCut {
+			break
+		}
+	}
+
+	newCutoff := feeds.FindCutoff(cutoff, pkgs)
+	pkgs = feeds.ApplyCutoff(pkgs, cutoff)
+
+	return pkgs, newCutoff, errs
+}
+
+func (feed Feed) GetName() string {
+	return FeedName
+}
+
+func (feed Feed) GetFeedOptions() feeds.FeedOptions {
+	return feed.options
+}

pkg/feeds/maven/maven_test.go

@@ -0,0 +1,113 @@
+package maven
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/ossf/package-feeds/pkg/feeds"
+	testutils "github.com/ossf/package-feeds/pkg/utils/test"
+)
+
+func TestMavenLatest(t *testing.T) {
+	t.Parallel()
+
+	handlers := map[string]testutils.HTTPHandlerFunc{
+		indexPath: mavenPackageResponse,
+	}
+	srv := testutils.HTTPServerMock(handlers)
+
+	feed, err := New(feeds.FeedOptions{})
+	if err != nil {
+		t.Fatalf("Failed to create Maven feed: %v", err)
+	}
+	feed.baseURL = srv.URL + "/api/internal/browse/components"
+
+	cutoff := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)
+	pkgs, gotCutoff, errs := feed.Latest(cutoff)
+
+	if len(errs) != 0 {
+		t.Fatalf("feed.Latest returned error: %v", err)
+	}
+
+	// Returned cutoff should match the newest package creation time of packages retrieved.
+	wantCutoff := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+	if gotCutoff.UTC().Sub(wantCutoff).Abs() > time.Second {
+		t.Errorf("Latest() cutoff %v, want %v", gotCutoff, wantCutoff)
+	}
+	if pkgs[0].Name != "com.github.example:project" {
+		t.Errorf("Unexpected package `%s` found in place of expected `com.github.example:project`", pkgs[0].Name)
+	}
+	if pkgs[0].Version != "1.0.0" {
+		t.Errorf("Unexpected version `%s` found in place of expected `1.0.0`", pkgs[0].Version)
+	}
+
+	for _, p := range pkgs {
+		if p.Type != FeedName {
+			t.Errorf("Feed type not set correctly in goproxy package following Latest()")
+		}
+	}
+}
+
+func TestMavenNotFound(t *testing.T) {
+	t.Parallel()
+
+	handlers := map[string]testutils.HTTPHandlerFunc{
+		indexPath: testutils.NotFoundHandlerFunc,
+	}
+	srv := testutils.HTTPServerMock(handlers)
+
+	feed, err := New(feeds.FeedOptions{})
+	if err != nil {
+		t.Fatalf("Failed to create Maven feed: %v", err)
+	}
+	feed.baseURL = srv.URL + "/api/internal/browse/components"
+
+	cutoff := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
+
+	_, gotCutoff, errs := feed.Latest(cutoff)
+	if cutoff != gotCutoff {
+		t.Error("feed.Latest() cutoff should be unchanged if an error is returned")
+	}
+	if len(errs) == 0 {
+		t.Fatalf("feed.Latest() was successful when an error was expected")
+	}
+}
+
+func mavenPackageResponse(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	responseJSON := `
+	{
+			"components": [
+					{
+							"id": "pkg:maven/com.github.example/project",
+							"type": "COMPONENT",
+							"namespace": "com.github.example",
+							"name": "project",
+							"version": "1.0.0",
+							"publishedEpochMillis": 946684800000,
+							"latestVersionInfo": {
+									"version": "1.0.0",
+									"timestampUnixWithMS": 946684800000
+							}
+					},
+					{
+						"id": "pkg:maven/com.github.example/project1",
+						"type": "COMPONENT",
+						"namespace": "com.github.example",
+						"name": "project",
+						"version": "1.0.0",
+						"publishedEpochMillis": null,
+						"latestVersionInfo": {
+								"version": "1.0.0",
+								"timestampUnixWithMS": 0
+						}
+				}
+			]
+	}
+	`
+	_, err := w.Write([]byte(responseJSON))
+	if err != nil {
+		http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError)
+	}
+}