Add package registry quota limits (#21584)

Related #20471

This PR adds global quota limits for the package registry. Settings for
individual users/orgs can be added in a seperate PR using the settings
table.

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

cmd/migrate_storage_test.go

@@ -44,8 +44,9 @@ func TestMigratePackages(t *testing.T) {
 		PackageFileInfo: packages_service.PackageFileInfo{
 			Filename: "a.go",
 		},
-		Data:   buf,
-		IsLead: true,
+		Creator: creator,
+		Data:    buf,
+		IsLead:  true,
 	})
 	assert.NoError(t, err)
 	assert.NotNil(t, v)

custom/conf/app.example.ini

@@ -2335,6 +2335,35 @@ ROUTER = console
 ;;
 ;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload`
 ;CHUNKED_UPLOAD_PATH = tmp/package-upload
+;;
+;; Maxmimum count of package versions a single owner can have (`-1` means no limits)
+;LIMIT_TOTAL_OWNER_COUNT = -1
+;; Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_TOTAL_OWNER_SIZE = -1
+;; Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_COMPOSER = -1
+;; Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_CONAN = -1
+;; Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_CONTAINER = -1
+;; Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_GENERIC = -1
+;; Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_HELM = -1
+;; Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_MAVEN = -1
+;; Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_NPM = -1
+;; Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_NUGET = -1
+;; Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_PUB = -1
+;; Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_PYPI = -1
+;; Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_RUBYGEMS = -1
+;; Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_VAGRANT = -1
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -1138,6 +1138,20 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 
 - `ENABLED`: **true**: Enable/Disable package registry capabilities
 - `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload`
+- `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maxmimum count of package versions a single owner can have (`-1` means no limits)
+- `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_COMPOSER`: **-1**: Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_CONAN`: **-1**: Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_CONTAINER`: **-1**: Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_GENERIC`: **-1**: Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_HELM`: **-1**: Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_MAVEN`: **-1**: Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_NPM`: **-1**: Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_NUGET`: **-1**: Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_PUB`: **-1**: Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_PYPI`: **-1**: Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_RUBYGEMS`: **-1**: Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_VAGRANT`: **-1**: Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 
 ## Mirror (`mirror`)
 

models/packages/package_file.go

@@ -199,3 +199,13 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag
 	count, err := sess.FindAndCount(&pfs)
 	return pfs, count, err
 }
+
+// CalculateBlobSize sums up all blob sizes matching the search options.
+// It does NOT respect the deduplication of blobs.
+func CalculateBlobSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {
+	return db.GetEngine(ctx).
+		Table("package_file").
+		Where(opts.toConds()).
+		Join("INNER", "package_blob", "package_blob.id = package_file.blob_id").
+		SumInt(new(PackageBlob), "size")
+}

models/packages/package_version.go

@@ -319,3 +319,12 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
 	count, err := sess.FindAndCount(&pvs)
 	return pvs, count, err
 }
+
+// CountVersions counts all versions of packages matching the search options
+func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) {
+	return db.GetEngine(ctx).
+		Where(opts.toConds()).
+		Table("package_version").
+		Join("INNER", "package", "package.id = package_version.package_id").
+		Count(new(PackageVersion))
+}

modules/setting/packages.go

@@ -5,11 +5,15 @@
 package setting
 
 import (
+	"math"
 	"net/url"
 	"os"
 	"path/filepath"
 
 	"code.gitea.io/gitea/modules/log"
+
+	"github.com/dustin/go-humanize"
+	ini "gopkg.in/ini.v1"
 )
 
 // Package registry settings
@@ -19,8 +23,24 @@ var (
 		Enabled           bool
 		ChunkedUploadPath string
 		RegistryHost      string
+
+		LimitTotalOwnerCount int64
+		LimitTotalOwnerSize  int64
+		LimitSizeComposer    int64
+		LimitSizeConan       int64
+		LimitSizeContainer   int64
+		LimitSizeGeneric     int64
+		LimitSizeHelm        int64
+		LimitSizeMaven       int64
+		LimitSizeNpm         int64
+		LimitSizeNuGet       int64
+		LimitSizePub         int64
+		LimitSizePyPI        int64
+		LimitSizeRubyGems    int64
+		LimitSizeVagrant     int64
 	}{
-		Enabled: true,
+		Enabled:              true,
+		LimitTotalOwnerCount: -1,
 	}
 )
 
@@ -43,4 +63,32 @@ func newPackages() {
 	if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
 		log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
 	}
+
+	Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
+	Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")
+	Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
+	Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
+	Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
+	Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
+	Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")
+	Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM")
+	Packages.LimitSizeNuGet = mustBytes(sec, "LIMIT_SIZE_NUGET")
+	Packages.LimitSizePub = mustBytes(sec, "LIMIT_SIZE_PUB")
+	Packages.LimitSizePyPI = mustBytes(sec, "LIMIT_SIZE_PYPI")
+	Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS")
+	Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")
+}
+
+func mustBytes(section *ini.Section, key string) int64 {
+	const noLimit = "-1"
+
+	value := section.Key(key).MustString(noLimit)
+	if value == noLimit {
+		return -1
+	}
+	bytes, err := humanize.ParseBytes(value)
+	if err != nil || bytes > math.MaxInt64 {
+		return -1
+	}
+	return int64(bytes)
 }

modules/setting/packages_test.go

@@ -0,0 +1,31 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	ini "gopkg.in/ini.v1"
+)
+
+func TestMustBytes(t *testing.T) {
+	test := func(value string) int64 {
+		sec, _ := ini.Empty().NewSection("test")
+		sec.NewKey("VALUE", value)
+
+		return mustBytes(sec, "VALUE")
+	}
+
+	assert.EqualValues(t, -1, test(""))
+	assert.EqualValues(t, -1, test("-1"))
+	assert.EqualValues(t, 0, test("0"))
+	assert.EqualValues(t, 1, test("1"))
+	assert.EqualValues(t, 10000, test("10000"))
+	assert.EqualValues(t, 1000000, test("1 mb"))
+	assert.EqualValues(t, 1048576, test("1mib"))
+	assert.EqualValues(t, 1782579, test("1.7mib"))
+	assert.EqualValues(t, -1, test("1 yib")) // too large
+}

routers/api/packages/composer/composer.go

@@ -235,16 +235,20 @@ func UploadPackage(ctx *context.Context) {
 			PackageFileInfo: packages_service.PackageFileInfo{
 				Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)),
 			},
-			Data:   buf,
-			IsLead: true,
+			Creator: ctx.Doer,
+			Data:    buf,
+			IsLead:  true,
 		},
 	)
 	if err != nil {
-		if err == packages_model.ErrDuplicatePackageVersion {
+		switch err {
+		case packages_model.ErrDuplicatePackageVersion:
 			apiError(ctx, http.StatusBadRequest, err)
-			return
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
 		}
-		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}
 

routers/api/packages/conan/conan.go

@@ -348,8 +348,9 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
 			Filename:     strings.ToLower(filename),
 			CompositeKey: fileKey,
 		},
-		Data:   buf,
-		IsLead: isConanfileFile,
+		Creator: ctx.Doer,
+		Data:    buf,
+		IsLead:  isConanfileFile,
 		Properties: map[string]string{
 			conan_module.PropertyRecipeUser:     rref.User,
 			conan_module.PropertyRecipeChannel:  rref.Channel,
@@ -416,11 +417,14 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
 		pfci,
 	)
 	if err != nil {
-		if err == packages_model.ErrDuplicatePackageFile {
+		switch err {
+		case packages_model.ErrDuplicatePackageFile:
 			apiError(ctx, http.StatusBadRequest, err)
-			return
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
 		}
-		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}
 

routers/api/packages/generic/generic.go

@@ -104,16 +104,20 @@ func UploadPackage(ctx *context.Context) {
 			PackageFileInfo: packages_service.PackageFileInfo{
 				Filename: filename,
 			},
-			Data:   buf,
-			IsLead: true,
+			Creator: ctx.Doer,
+			Data:    buf,
+			IsLead:  true,
 		},
 	)
 	if err != nil {
-		if err == packages_model.ErrDuplicatePackageFile {
+		switch err {
+		case packages_model.ErrDuplicatePackageFile:
 			apiError(ctx, http.StatusConflict, err)
-			return
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
 		}
-		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}
 

routers/api/packages/helm/helm.go

@@ -186,17 +186,21 @@ func UploadPackage(ctx *context.Context) {
 			PackageFileInfo: packages_service.PackageFileInfo{
 				Filename: createFilename(metadata),
 			},
+			Creator:           ctx.Doer,
 			Data:              buf,
 			IsLead:            true,
 			OverwriteExisting: true,
 		},
 	)
 	if err != nil {
-		if err == packages_model.ErrDuplicatePackageVersion {
+		switch err {
+		case packages_model.ErrDuplicatePackageVersion:
 			apiError(ctx, http.StatusConflict, err)
-			return
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
 		}
-		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}
 

routers/api/packages/maven/maven.go

@@ -266,6 +266,7 @@ func UploadPackageFile(ctx *context.Context) {
 		PackageFileInfo: packages_service.PackageFileInfo{
 			Filename: params.Filename,
 		},
+		Creator:           ctx.Doer,
 		Data:              buf,
 		IsLead:            false,
 		OverwriteExisting: params.IsMeta,
@@ -312,11 +313,14 @@ func UploadPackageFile(ctx *context.Context) {
 		pfci,
 	)
 	if err != nil {
-		if err == packages_model.ErrDuplicatePackageFile {
+		switch err {
+		case packages_model.ErrDuplicatePackageFile:
 			apiError(ctx, http.StatusBadRequest, err)
-			return
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
 		}
-		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}
 

routers/api/packages/npm/npm.go

@@ -180,16 +180,20 @@ func UploadPackage(ctx *context.Context) {
 			PackageFileInfo: packages_service.PackageFileInfo{
 				Filename: npmPackage.Filename,
 			},
-			Data:   buf,
-			IsLead: true,
+			Creator: ctx.Doer,
+			Data:    buf,
+			IsLead:  true,
 		},
 	)
 	if err != nil {
-		if err == packages_model.ErrDuplicatePackageVersion {
+		switch err {
+		case packages_model.ErrDuplicatePackageVersion:
 			apiError(ctx, http.StatusBadRequest, err)
-			return
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
 		}
-		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}