diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 3c478b1c0288c..264b4612fb78a 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -48,10 +48,11 @@ const maxNuspecFileSize = 3 * 1024 * 1024 // Package represents a Nuget package type Package struct { - PackageType PackageType - ID string - Version string - Metadata *Metadata + PackageType PackageType + ID string + Version string + Metadata *Metadata + NuspecContent *bytes.Buffer } // Metadata represents the metadata of a Nuget package @@ -130,8 +131,9 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) { // ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package func ParseNuspecMetaData(r io.Reader) (*Package, error) { + var nuspecBuf bytes.Buffer var p nuspecPackage - if err := xml.NewDecoder(r).Decode(&p); err != nil { + if err := xml.NewDecoder(io.TeeReader(r, &nuspecBuf)).Decode(&p); err != nil { return nil, err } @@ -182,10 +184,11 @@ func ParseNuspecMetaData(r io.Reader) (*Package, error) { } } return &Package{ - PackageType: packageType, - ID: p.Metadata.ID, - Version: toNormalizedVersion(v), - Metadata: m, + PackageType: packageType, + ID: p.Metadata.ID, + Version: toNormalizedVersion(v), + Metadata: m, + NuspecContent: &nuspecBuf, }, nil } diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index c28bc6c9d9233..ab1b464af5f32 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -388,7 +388,8 @@ func EnumeratePackageVersionsV3(ctx *context.Context) { ctx.JSON(http.StatusOK, resp) } -// https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg +// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec +// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg func DownloadPackageFile(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") @@ -431,7 +432,7 @@ func UploadPackage(ctx *context.Context) { return } - _, _, err := packages_service.CreatePackageAndAddFile( + pv, _, err := packages_service.CreatePackageAndAddFile( ctx, &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ @@ -465,6 +466,35 @@ func UploadPackage(ctx *context.Context) { return } + nuspecBuf, err := packages_module.CreateHashedBufferFromReaderWithSize(np.NuspecContent, np.NuspecContent.Len()) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer nuspecBuf.Close() + + _, err = packages_service.AddFileToPackageVersionInternal( + ctx, + pv, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.nuspec", np.ID)), + }, + Creator: ctx.Doer, + Data: nuspecBuf, + IsLead: false, + }, + ) + if err != nil { + switch err { + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.Status(http.StatusCreated) } diff --git a/services/doctor/packages_nuget.go b/services/doctor/packages_nuget.go new file mode 100644 index 0000000000000..7f736b6e2a971 --- /dev/null +++ b/services/doctor/packages_nuget.go @@ -0,0 +1,171 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package doctor + +import ( + "context" + "fmt" + "slices" + "strings" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + nuget_module "code.gitea.io/gitea/modules/packages/nuget" + packages_service "code.gitea.io/gitea/services/packages" +) + +func init() { + Register(&Check{ + Title: "Extract Nuget Nuspec Files to content store", + Name: "packages-nuget-nuspec", + IsDefault: false, + Run: PackagesNugetNuspecCheck, + Priority: 15, + }) +} + +// getAllUsers returns a slice of all users and organizations found in DB. +func getAllUsers(ctx context.Context) ([]*user.User, error) { + users := make([]*user.User, 0) + return users, db.GetEngine(ctx).OrderBy("id").Find(&users) +} + +func PackagesNugetNuspecCheck(ctx context.Context, logger log.Logger, autofix bool) error { + users, err := getAllUsers(ctx) + userMap := make(map[int64]*user.User, len(users)) + + for _, u := range users { + userMap[u.ID] = u + } + + if err != nil { + logger.Error("Failed to get users: %v", err) + return err + } + + logger.Info("Found %d users", len(users)) + + fixed := 0 + errors := 0 + + for _, user := range users { + pkgs, err := packages.GetPackagesByType(ctx, user.ID, packages.TypeNuGet) + if err != nil { + logger.Error("Failed to get NuGet packages for owner %s: %v", user.Name, err) + continue + } + + logger.Info("Found %d NuGet packages for owner %s", len(pkgs), user.Name) + + for _, pkg := range pkgs { + pvs, _, err := packages.SearchVersions(ctx, &packages.PackageSearchOptions{ + Type: packages.TypeNuGet, + PackageID: pkg.ID, + }) + if err != nil { + // Should never happen + logger.Error("Failed to search for versions for package %s: %v", pkg.Name, err) + continue + } + + logger.Info("Found %d versions for package %s", pkg.Name, user.Name) + + for _, pv := range pvs { + + pfs, err := packages.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + logger.Error("Failed to get files for package version %s %d: %v", pkg.Name, pv.Version, err) + errors++ + continue + } + + if slices.IndexFunc(pfs, func(pf *packages.PackageFile) bool { return strings.HasSuffix(pf.LowerName, ".nuspec") }) >= 0 { + logger.Debug("Nuspec file already exists for %s %d", pkg.Name, pv.Version) + continue + } + + nuspecIdx := slices.IndexFunc(pfs, func(pf *packages.PackageFile) bool { return pf.IsLead }) + + if nuspecIdx < 0 { + logger.Error("Missing nupkg file for %s %d", pkg.Name, pv.Version) + errors++ + continue + } + + pf := pfs[nuspecIdx] + + creator, ok := userMap[pv.CreatorID] + if !ok { + logger.Warn("Failed to find creator for %s %d", pkg.Name, pv.Version) + creator = user + } + + logger.Info("Missing nuspec file found for %s %d", pkg.Name, pv.Version) + fixed++ + + if autofix { + s, _, _, err := packages_service.GetPackageFileStream(ctx, pf) + if err != nil { + logger.Error("Failed to get file stream for %s %d: %v", pkg.Name, pv.Version, err) + errors++ + continue + } + defer s.Close() + + buf, err := packages_module.CreateHashedBufferFromReader(s) + if err != nil { + logger.Error("Failed to create hashed buffer for nupkg from reader for %s %d: %v", pkg.Name, pv.Version, err) + errors++ + continue + } + defer buf.Close() + + np, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) + if err != nil { + logger.Error("Failed to parse package metadata for %s %d: %v", pkg.Name, pv.Version, err) + errors++ + continue + } + + nuspecBuf, err := packages_module.CreateHashedBufferFromReaderWithSize(np.NuspecContent, np.NuspecContent.Len()) + if err != nil { + logger.Error("Failed to create hashed buffer for nuspec from reader for %s %d: %v", pkg.Name, pv.Version, err) + errors++ + continue + } + defer nuspecBuf.Close() + + _, err = packages_service.AddFileToPackageVersionInternal( + ctx, + pv, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fmt.Sprintf("%s.nuspec", pkg.LowerName), + }, + Creator: creator, + Data: nuspecBuf, + IsLead: false, + }, + ) + if err != nil { + logger.Error("Failed to add nuspec file for %s %d: %v", pkg.Name, pv.Version, err) + errors++ + } + } + } + } + } + + if fixed > 0 { + logger.Info("Fixed %d NuGet packages by extracting nuspec files", fixed) + } + if errors > 0 { + logger.Info("Failed to fix %d nuspec files", errors) + } + + return nil +} diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go index 20dafd5cc799f..991f37fe742dc 100644 --- a/tests/integration/api_packages_nuget_test.go +++ b/tests/integration/api_packages_nuget_test.go @@ -112,6 +112,20 @@ func TestPackageNuGet(t *testing.T) { return &buf } + nuspec := ` + + + ` + packageName + ` + ` + packageVersion + ` + ` + packageAuthors + ` + ` + packageDescription + ` + + + + + + + ` content, _ := io.ReadAll(createPackage(packageName, packageVersion)) url := fmt.Sprintf("/api/packages/%s/nuget", user.Name) @@ -224,7 +238,7 @@ func TestPackageNuGet(t *testing.T) { pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet) assert.NoError(t, err) - assert.Len(t, pvs, 1) + assert.Len(t, pvs, 1, "Should have one version") pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) @@ -235,7 +249,7 @@ func TestPackageNuGet(t *testing.T) { pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) - assert.Len(t, pfs, 1) + assert.Len(t, pfs, 2, "Should have 2 files: nuget and nuspec") assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) assert.True(t, pfs[0].IsLead) @@ -302,16 +316,27 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) - assert.Len(t, pfs, 3) + assert.Len(t, pfs, 4, "Should have 4 files: nupkg, snupkg, nuspec and pdb") for _, pf := range pfs { switch pf.Name { case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion): + assert.True(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(414), pb.Size) case fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion): assert.False(t, pf.IsLead) pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) assert.NoError(t, err) assert.Equal(t, int64(616), pb.Size) + case fmt.Sprintf("%s.nuspec", packageName): + assert.False(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(453), pb.Size) case symbolFilename: assert.False(t, pf.IsLead) @@ -353,6 +378,12 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) assert.Equal(t, content, resp.Body.Bytes()) + req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.nuspec", url, packageName, packageVersion, packageName)). + AddBasicAuth(user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, nuspec, resp.Body.String()) + checkDownloadCount(1) req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)). diff --git a/tests/integration/doctor_packages_nuget_test.go b/tests/integration/doctor_packages_nuget_test.go new file mode 100644 index 0000000000000..0bcbf1664f0e9 --- /dev/null +++ b/tests/integration/doctor_packages_nuget_test.go @@ -0,0 +1,116 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "strings" + "testing" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + doctor "code.gitea.io/gitea/services/doctor" + packages_service "code.gitea.io/gitea/services/packages" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestDoctorPackagesNuget(t *testing.T) { + defer tests.PrepareTestEnv(t, 1)() + + logger := log.GetLogger("doctor") + + ctx := db.DefaultContext + + packageName := "test.package" + packageVersion := "1.0.3" + packageAuthors := "KN4CK3R" + packageDescription := "Gitea Test Package" + + createPackage := func(id, version string) io.Reader { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create("package.nuspec") + w.Write([]byte(` + + + ` + id + ` + ` + version + ` + ` + packageAuthors + ` + ` + packageDescription + ` + + + + + + + `)) + archive.Close() + return &buf + } + + pkg := createPackage(packageName, packageVersion) + + pkgBuf, err := packages_module.CreateHashedBufferFromReader(pkg) + assert.NoError(t, err, "Error creating hashed buffer from nupkg") + defer pkgBuf.Close() + + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + assert.NoError(t, err, "Error getting user by ID 1") + + t.Run("PackagesNugetNuspecCheck", func(t *testing.T) { + pi := &packages_service.PackageInfo{ + Owner: doer, + PackageType: packages_model.TypeNuGet, + Name: packageName, + Version: packageVersion, + } + _, _, err := packages_service.CreatePackageAndAddFile( + ctx, + &packages_service.PackageCreationInfo{ + PackageInfo: *pi, + SemverCompatible: true, + Creator: doer, + Metadata: nil, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion)), + }, + Creator: doer, + Data: pkgBuf, + IsLead: true, + }, + ) + assert.NoError(t, err, "Error creating package and adding file") + + assert.NoError(t, doctor.PackagesNugetNuspecCheck(ctx, logger, true), "Doctor check failed") + + s, _, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: doer, + PackageType: packages_model.TypeNuGet, + Name: packageName, + Version: packageVersion, + }, + &packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.nuspec", packageName)), + }, + ) + + assert.NoError(t, err, "Error getting nuspec file stream by package name and version") + defer s.Close() + + assert.Equal(t, fmt.Sprintf("%s.nuspec", packageName), pf.Name, "Not a nuspec") + }) +}