Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Package registry changes #19305

Merged
merged 21 commits into from
Apr 6, 2022
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 30 additions & 3 deletions integrations/api_packages_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@ import (
"fmt"
"net/http"
"testing"
"time"

"code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
container_model "code.gitea.io/gitea/models/packages/container"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
api "code.gitea.io/gitea/modules/structs"
packages_service "code.gitea.io/gitea/services/packages"

"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -43,7 +47,7 @@ func TestPackageAPI(t *testing.T) {
DecodeJSON(t, resp, &apiPackages)

assert.Len(t, apiPackages, 1)
assert.Equal(t, string(packages.TypeGeneric), apiPackages[0].Type)
assert.Equal(t, string(packages_model.TypeGeneric), apiPackages[0].Type)
assert.Equal(t, packageName, apiPackages[0].Name)
assert.Equal(t, packageVersion, apiPackages[0].Version)
assert.NotNil(t, apiPackages[0].Creator)
Expand All @@ -62,7 +66,7 @@ func TestPackageAPI(t *testing.T) {
var p *api.Package
DecodeJSON(t, resp, &p)

assert.Equal(t, string(packages.TypeGeneric), p.Type)
assert.Equal(t, string(packages_model.TypeGeneric), p.Type)
assert.Equal(t, packageName, p.Name)
assert.Equal(t, packageVersion, p.Version)
assert.NotNil(t, p.Creator)
Expand Down Expand Up @@ -100,3 +104,26 @@ func TestPackageAPI(t *testing.T) {
MakeRequest(t, req, http.StatusNoContent)
})
}

func TestPackageCleanup(t *testing.T) {
defer prepareTestEnv(t)()

time.Sleep(time.Second)
6543 marked this conversation as resolved.
Show resolved Hide resolved

pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0))
assert.NoError(t, err)
assert.NotEmpty(t, pbs)

_, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion)
assert.NoError(t, err)

err = packages_service.Cleanup(nil, time.Duration(0))
assert.NoError(t, err)

pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0))
assert.NoError(t, err)
assert.Empty(t, pbs)

_, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion)
assert.ErrorIs(t, err, packages_model.ErrPackageNotExist)
}
6 changes: 3 additions & 3 deletions models/packages/conan/references.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,14 @@ func findPropertyValues(ctx context.Context, propertyName string, ownerID int64,
in2 := builder.
Select("package_file.id").
From("package_file").
Join("INNER", "package_version", "package_version.id = package_file.version_id").
Join("INNER", "package", "package.id = package_version.package_id").
InnerJoin("package_version", "package_version.id = package_file.version_id").
InnerJoin("package", "package.id = package_version.package_id").
Where(cond)

query := builder.
Select("package_property.value, MAX(package_file.created_unix) AS created_unix").
From("package_property").
Join("INNER", "package_file", "package_file.id = package_property.ref_id").
InnerJoin("package_file", "package_file.id = package_property.ref_id").
Where(builder.Eq{"package_property.name": propertyName}.And(builder.In("package_property.ref_id", in2))).
GroupBy("package_property.value").
OrderBy("created_unix DESC")
Expand Down
4 changes: 2 additions & 2 deletions models/packages/conan/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ func SearchRecipes(ctx context.Context, opts *RecipeSearchOptions) ([]string, er
query := builder.
Select("package.name, package_version.version, package_file.id").
From("package_file").
Join("INNER", "package_version", "package_version.id = package_file.version_id").
Join("INNER", "package", "package.id = package_version.package_id").
InnerJoin("package_version", "package_version.id = package_file.version_id").
InnerJoin("package", "package.id = package_version.package_id").
Where(cond)

results := make([]struct {
Expand Down
8 changes: 5 additions & 3 deletions models/packages/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,13 +190,15 @@ func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]
// DeletePackagesIfUnreferenced deletes a package if there are no associated versions
func DeletePackagesIfUnreferenced(ctx context.Context) error {
in := builder.
Select("package_version.package_id").
Select("package.id").
lunny marked this conversation as resolved.
Show resolved Hide resolved
From("package").
Join("LEFT", "package_version", "package_version.package_id = package.id").
LeftJoin("package_version", "package_version.package_id = package.id").
Where(builder.Expr("package_version.id IS NULL"))

_, err := db.GetEngine(ctx).
Where(builder.In("package.id", in)).
// double select workaround for MySQL
// https://stackoverflow.com/questions/4471277/mysql-delete-from-with-subquery-as-condition
Where(builder.In("package.id", builder.Select("id").From(in, "temp"))).
Delete(&Package{})

return err
Expand Down
2 changes: 1 addition & 1 deletion models/packages/package_blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func FindExpiredUnreferencedBlobs(ctx context.Context, olderThan time.Duration)
pbs := make([]*PackageBlob, 0, 10)
return pbs, db.GetEngine(ctx).
Table("package_blob").
Join("LEFT OUTER", "package_file", "package_file.blob_id = package_blob.id").
Join("LEFT", "package_file", "package_file.blob_id = package_blob.id").
6543 marked this conversation as resolved.
Show resolved Hide resolved
Where("package_file.id IS NULL AND package_blob.created_unix < ?", time.Now().Add(-olderThan).Unix()).
Find(&pbs)
}
Expand Down
2 changes: 1 addition & 1 deletion models/packages/package_file.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func (opts *PackageFileSearchOptions) toConds() builder.Cond {
in := builder.
Select("package_version.id").
From("package_version").
Join("INNER", "package", "package.id = package_version.package_id").
InnerJoin("package", "package.id = package_version.package_id").
Where(versionCond)

cond = cond.And(builder.In("package_file.version_id", in))
Expand Down
173 changes: 81 additions & 92 deletions models/packages/package_version.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,13 @@ import (

"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"

"xorm.io/builder"
)

var (
// ErrDuplicatePackageVersion indicates a duplicated package version error
ErrDuplicatePackageVersion = errors.New("Package version does exist already")
// ErrPackageVersionNotExist indicates a package version not exist error
ErrPackageVersionNotExist = errors.New("Package version does not exist")
)
// ErrDuplicatePackageVersion indicates a duplicated package version error
var ErrDuplicatePackageVersion = errors.New("Package version already exists")

func init() {
db.RegisterModel(new(PackageVersion))
Expand Down Expand Up @@ -99,75 +96,49 @@ func GetInternalVersionByNameAndVersion(ctx context.Context, ownerID int64, pack
}

func getVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version string, isInternal bool) (*PackageVersion, error) {
var cond builder.Cond = builder.Eq{
"package.owner_id": ownerID,
"package.type": packageType,
"package.lower_name": strings.ToLower(name),
"package_version.is_internal": isInternal,
}
pv := &PackageVersion{
LowerVersion: strings.ToLower(version),
}
has, err := db.GetEngine(ctx).
Join("INNER", "package", "package.id = package_version.package_id").
Where(cond).
Get(pv)
pvs, _, err := SearchVersions(ctx, &PackageSearchOptions{
OwnerID: ownerID,
Type: packageType,
Name: SearchValue{
ExactMatch: true,
Value: name,
},
Version: SearchValue{
ExactMatch: true,
Value: version,
},
IsInternal: isInternal,
Paginator: db.NewAbsoluteListOptions(0, 1),
})
if err != nil {
return nil, err
}
if !has {
if len(pvs) == 0 {
return nil, ErrPackageNotExist
}

return pv, nil
return pvs[0], nil
}

// GetVersionsByPackageType gets all versions of a specific type
func GetVersionsByPackageType(ctx context.Context, ownerID int64, packageType Type) ([]*PackageVersion, error) {
var cond builder.Cond = builder.Eq{
"package.owner_id": ownerID,
"package.type": packageType,
"package_version.is_internal": false,
}

pvs := make([]*PackageVersion, 0, 10)
return pvs, db.GetEngine(ctx).
Where(cond).
Join("INNER", "package", "package.id = package_version.package_id").
Find(&pvs)
pvs, _, err := SearchVersions(ctx, &PackageSearchOptions{
OwnerID: ownerID,
Type: packageType,
})
return pvs, err
}

// GetVersionsByPackageName gets all versions of a specific package
func GetVersionsByPackageName(ctx context.Context, ownerID int64, packageType Type, name string) ([]*PackageVersion, error) {
var cond builder.Cond = builder.Eq{
"package.owner_id": ownerID,
"package.type": packageType,
"package.lower_name": strings.ToLower(name),
"package_version.is_internal": false,
}

pvs := make([]*PackageVersion, 0, 10)
return pvs, db.GetEngine(ctx).
Where(cond).
Join("INNER", "package", "package.id = package_version.package_id").
Find(&pvs)
}

// GetVersionsByFilename gets all versions which are linked to a filename
func GetVersionsByFilename(ctx context.Context, ownerID int64, packageType Type, filename string) ([]*PackageVersion, error) {
var cond builder.Cond = builder.Eq{
"package.owner_id": ownerID,
"package.type": packageType,
"package_file.lower_name": strings.ToLower(filename),
"package_version.is_internal": false,
}

pvs := make([]*PackageVersion, 0, 10)
return pvs, db.GetEngine(ctx).
Where(cond).
Join("INNER", "package_file", "package_file.version_id = package_version.id").
Join("INNER", "package", "package.id = package_version.package_id").
Find(&pvs)
pvs, _, err := SearchVersions(ctx, &PackageSearchOptions{
OwnerID: ownerID,
Type: packageType,
Name: SearchValue{
ExactMatch: true,
Value: name,
},
})
return pvs, err
}

// DeleteVersionByID deletes a version by id
Expand All @@ -183,21 +154,32 @@ func HasVersionFileReferences(ctx context.Context, versionID int64) (bool, error
})
}

// SearchValue describes a value to search
// If ExactMatch is true, the field must match the value otherwise a LIKE search is performed.
type SearchValue struct {
Value string
ExactMatch bool
}

// PackageSearchOptions are options for SearchXXX methods
// Besides IsInternal are all fields optional and are not used if they have their default value (nil, "", 0)
type PackageSearchOptions struct {
OwnerID int64
RepoID int64
Type string
PackageID int64
QueryName string
QueryVersion string
Properties map[string]string
Sort string
OwnerID int64
RepoID int64
Type Type
PackageID int64
Name SearchValue // only results with the specific name are found
Version SearchValue // only results with the specific version are found
Properties map[string]string // only results are found which contain all listed version properties with the specific value
IsInternal bool
HasFileWithName string // only results are found which are associated with a file with the specific name
HasFiles util.OptionalBool // only results are found which have associated files
Sort string
db.Paginator
}

func (opts *PackageSearchOptions) toConds() builder.Cond {
var cond builder.Cond = builder.Eq{"package_version.is_internal": false}
var cond builder.Cond = builder.Eq{"package_version.is_internal": opts.IsInternal}

if opts.OwnerID != 0 {
cond = cond.And(builder.Eq{"package.owner_id": opts.OwnerID})
Expand All @@ -211,11 +193,19 @@ func (opts *PackageSearchOptions) toConds() builder.Cond {
if opts.PackageID != 0 {
cond = cond.And(builder.Eq{"package.id": opts.PackageID})
}
if opts.QueryName != "" {
cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.QueryName)})
if opts.Name.Value != "" {
if opts.Name.ExactMatch {
cond = cond.And(builder.Eq{"package.lower_name": strings.ToLower(opts.Name.Value)})
} else {
cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Name.Value)})
}
}
if opts.QueryVersion != "" {
cond = cond.And(builder.Like{"package_version.lower_version", strings.ToLower(opts.QueryVersion)})
if opts.Version.Value != "" {
if opts.Version.ExactMatch {
cond = cond.And(builder.Eq{"package_version.lower_version": strings.ToLower(opts.Version.Value)})
} else {
cond = cond.And(builder.Like{"package_version.lower_version", strings.ToLower(opts.Version.Value)})
}
}

if len(opts.Properties) != 0 {
Expand All @@ -238,6 +228,22 @@ func (opts *PackageSearchOptions) toConds() builder.Cond {
})
}

if opts.HasFileWithName != "" {
fileCond := builder.Expr("package_file.version_id = package_version.id").And(builder.Eq{"package_file.lower_name": strings.ToLower(opts.HasFileWithName)})

cond = cond.And(builder.Exists(builder.Select("package_file.id").From("package_file").Where(fileCond)))
}

if !opts.HasFiles.IsNone() {
var filesCond builder.Cond = builder.Exists(builder.Select("package_file.id").From("package_file").Where(builder.Expr("package_file.version_id = package_version.id")))

if opts.HasFiles.IsFalse() {
filesCond = builder.Not{filesCond}
}

cond = cond.And(filesCond)
}

return cond
}

Expand Down Expand Up @@ -297,20 +303,3 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
count, err := sess.FindAndCount(&pvs)
return pvs, count, err
}

// FindVersionsByPropertyNameAndValue gets all package versions which are associated with a specific property + value
func FindVersionsByPropertyNameAndValue(ctx context.Context, packageID int64, name, value string) ([]*PackageVersion, error) {
var cond builder.Cond = builder.Eq{
"package_property.ref_type": PropertyTypeVersion,
"package_property.name": name,
"package_property.value": value,
"package_version.package_id": packageID,
"package_version.is_internal": false,
}

pvs := make([]*PackageVersion, 0, 5)
return pvs, db.GetEngine(ctx).
Where(cond).
Join("INNER", "package_property", "package_property.ref_id = package_version.id").
Find(&pvs)
}
4 changes: 2 additions & 2 deletions routers/api/packages/composer/composer.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ func SearchPackages(ctx *context.Context) {

opts := &packages_model.PackageSearchOptions{
OwnerID: ctx.Package.Owner.ID,
Type: string(packages_model.TypeComposer),
QueryName: ctx.FormTrim("q"),
Type: packages_model.TypeComposer,
Name: packages_model.SearchValue{Value: ctx.FormTrim("q")},
Paginator: &paginator,
}
if ctx.FormTrim("type") != "" {
Expand Down
7 changes: 6 additions & 1 deletion routers/api/packages/npm/npm.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,12 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo
}
defer committer.Close()

pvs, err := packages_model.FindVersionsByPropertyNameAndValue(ctx, pv.PackageID, npm_module.TagProperty, tag)
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
PackageID: pv.PackageID,
Properties: map[string]string{
npm_module.TagProperty: tag,
},
})
if err != nil {
return err
}
Expand Down
Loading