Adjust gitea doctor --run storages to check all storage types

models/git/lfs.go

@@ -236,8 +236,8 @@ func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) {
 }
 
 // LFSObjectIsAssociated checks if a provided Oid is associated
-func LFSObjectIsAssociated(oid string) (bool, error) {
-	return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
+func LFSObjectIsAssociated(ctx context.Context, oid string) (bool, error) {
+	return db.GetEngine(ctx).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
 }
 
 // LFSAutoAssociate auto associates accessible LFSMetaObjects

models/packages/package_blob.go

@@ -62,6 +62,13 @@ func GetBlobByID(ctx context.Context, blobID int64) (*PackageBlob, error) {
 	return pb, nil
 }
 
+// ExistPackageBlobWithSHA returns if a package blob exists with the provided sha
+func ExistPackageBlobWithSHA(ctx context.Context, blobSha256 string) (bool, error) {
+	return db.GetEngine(ctx).Exist(&PackageBlob{
+		HashSHA256: blobSha256,
+	})
+}
+
 // FindExpiredUnreferencedBlobs gets all blobs without associated files older than the specific duration
 func FindExpiredUnreferencedBlobs(ctx context.Context, olderThan time.Duration) ([]*PackageBlob, error) {
 	pbs := make([]*PackageBlob, 0, 10)

models/repo/archiver.go

@@ -7,11 +7,14 @@ package repo
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
 
 	"xorm.io/builder"
 )
@@ -65,6 +68,31 @@ func GetRepoArchiver(ctx context.Context, repoID int64, tp git.ArchiveType, comm
 	return nil, nil
 }
 
+// ExistsRepoArchiverWithStoragePath checks if there is a RepoArchiver for a given storage path
+func ExistsRepoArchiverWithStoragePath(ctx context.Context, pth string) (bool, error) {
+	// fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String())
+	parts := strings.SplitN(pth, "/", 3)
+	if len(parts) != 3 {
+		return false, util.SilentWrap{Message: fmt.Sprintf("invalid storage path: %s", pth), Err: util.ErrInvalidArgument}
+	}
+	repoID, err := strconv.ParseInt(parts[0], 10, 64)
+	if err != nil {
+		return false, util.SilentWrap{Message: fmt.Sprintf("invalid storage path: %s", pth), Err: util.ErrInvalidArgument}
+	}
+	nameExts := strings.SplitN(parts[2], ".", 2)
+	if len(nameExts) != 2 {
+		return false, util.SilentWrap{Message: fmt.Sprintf("invalid storage path: %s", pth), Err: util.ErrInvalidArgument}
+	}
+
+	archiver := &RepoArchiver{
+		RepoID:   repoID,
+		CommitID: parts[1] + nameExts[0],
+		Type:     git.ToArchiveType(nameExts[1]),
+	}
+
+	return db.GetEngine(ctx).Exist(&archiver)
+}
+
 // AddRepoArchiver adds an archiver
 func AddRepoArchiver(ctx context.Context, archiver *RepoArchiver) error {
 	_, err := db.GetEngine(ctx).Insert(archiver)

models/repo/attachment.go

@@ -122,9 +122,9 @@ func GetAttachmentsByUUIDs(ctx context.Context, uuids []string) ([]*Attachment,
 	return attachments, db.GetEngine(ctx).In("uuid", uuids).Find(&attachments)
 }
 
-// ExistAttachmentsByUUID returns true if attachment is exist by given UUID
-func ExistAttachmentsByUUID(uuid string) (bool, error) {
-	return db.GetEngine(db.DefaultContext).Where("`uuid`=?", uuid).Exist(new(Attachment))
+// ExistAttachmentsByUUID returns true if attachment exists with the given UUID
+func ExistAttachmentsByUUID(ctx context.Context, uuid string) (bool, error) {
+	return db.GetEngine(ctx).Where("`uuid`=?", uuid).Exist(new(Attachment))
 }
 
 // GetAttachmentsByIssueID returns all attachments of an issue.

models/repo/avatar.go

@@ -24,6 +24,11 @@ func (repo *Repository) CustomAvatarRelativePath() string {
 	return repo.Avatar
 }
 
+// ExistRepoWithAvatar returns true if there is a user with this Avatar
+func ExistRepoWithAvatar(ctx context.Context, avatar string) (bool, error) {
+	return db.GetEngine(ctx).Where("`avatar`=?", avatar).Exist(new(Repository))
+}
+
 // RelAvatarLink returns a relative link to the repository's avatar.
 func (repo *Repository) RelAvatarLink() string {
 	return repo.relAvatarLink(db.DefaultContext)

models/user/avatar.go

@@ -111,3 +111,8 @@ func (u *User) IsUploadAvatarChanged(data []byte) bool {
 	avatarID := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%x", u.ID, md5.Sum(data)))))
 	return u.Avatar != avatarID
 }
+
+// ExistUserWithAvatar returns true if there is a user with this Avatar
+func ExistUserWithAvatar(ctx context.Context, avatar string) (bool, error) {
+	return db.GetEngine(ctx).Where("`avatar`=?", avatar).Exist(new(User))
+}

modules/doctor/storage.go

@@ -6,71 +6,247 @@ package doctor
 
 import (
 	"context"
+	"errors"
+	"io/fs"
+	"strings"
 
+	"code.gitea.io/gitea/models/git"
+	"code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/models/repo"
 	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/util"
 )
 
-func checkAttachmentStorageFiles(logger log.Logger, autofix bool) error {
-	var total, garbageNum int
-	var deletePaths []string
-	if err := storage.Attachments.IterateObjects(func(p string, obj storage.Object) error {
+type commonStorageCheckOptions struct {
+	storer       storage.ObjectStorage
+	isAssociated func(path string, obj storage.Object, stat fs.FileInfo) (bool, error)
+	name         string
+}
+
+func commonCheckStorage(ctx context.Context, logger log.Logger, autofix bool, opts *commonStorageCheckOptions) error {
+	totalCount, unassociatedCount := 0, 0
+	totalSize, unassociatedSize := int64(0), int64(0)
+
+	var pathsToDelete []string
+	if err := opts.storer.IterateObjects(func(p string, obj storage.Object) error {
 		defer obj.Close()
 
-		total++
+		totalCount++
 		stat, err := obj.Stat()
 		if err != nil {
 			return err
 		}
-		exist, err := repo_model.ExistAttachmentsByUUID(stat.Name())
+		totalSize += stat.Size()
+
+		associated, err := opts.isAssociated(p, obj, stat)
 		if err != nil {
 			return err
 		}
-		if !exist {
-			garbageNum++
+		if !associated {
+			unassociatedCount++
+			unassociatedSize += stat.Size()
 			if autofix {
-				deletePaths = append(deletePaths, p)
+				pathsToDelete = append(pathsToDelete, p)
 			}
 		}
 		return nil
 	}); err != nil {
-		logger.Error("storage.Attachments.IterateObjects failed: %v", err)
+		logger.Error("Error whilst iterating %s storage: %v", opts.name, err)
 		return err
 	}
 
-	if garbageNum > 0 {
+	if unassociatedCount > 0 {
 		if autofix {
 			var deletedNum int
-			for _, p := range deletePaths {
-				if err := storage.Attachments.Delete(p); err != nil {
-					log.Error("Delete attachment %s failed: %v", p, err)
+			for _, p := range pathsToDelete {
+				if err := opts.storer.Delete(p); err != nil {
+					log.Error("Error whilst deleting %s from %s storage: %v", p, opts.name, err)
 				} else {
 					deletedNum++
 				}
 			}
-			logger.Info("%d missed information attachment detected, %d deleted.", garbageNum, deletedNum)
+			logger.Info("Deleted %d/%d unassociated %s(s)", deletedNum, unassociatedCount, opts.name)
 		} else {
-			logger.Warn("Checked %d attachment, %d missed information.", total, garbageNum)
+			logger.Warn("Found %d/%d (%s/%s) unassociated %s(s)", unassociatedCount, totalCount, base.FileSize(unassociatedSize), base.FileSize(totalSize), opts.name)
 		}
+	} else {
+		logger.Info("Found %d (%s) %s(s)", totalCount, base.FileSize(totalSize), opts.name)
 	}
 	return nil
 }
 
-func checkStorageFiles(ctx context.Context, logger log.Logger, autofix bool) error {
-	if err := storage.Init(); err != nil {
-		logger.Error("storage.Init failed: %v", err)
-		return err
+type storageCheckOptions struct {
+	All          bool
+	Attachments  bool
+	LFS          bool
+	Avatars      bool
+	RepoAvatars  bool
+	RepoArchives bool
+	Packages     bool
+}
+
+func checkStorage(opts *storageCheckOptions) func(ctx context.Context, logger log.Logger, autofix bool) error {
+	return func(ctx context.Context, logger log.Logger, autofix bool) error {
+		if err := storage.Init(); err != nil {
+			logger.Error("storage.Init failed: %v", err)
+			return err
+		}
+
+		if opts.Attachments || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Attachments,
+					isAssociated: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						return repo_model.ExistAttachmentsByUUID(ctx, stat.Name())
+					},
+					name: "attachment",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.LFS || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.LFS,
+					isAssociated: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						// The oid of an LFS stored object is the name but with all the path.Separators removed
+						oid := strings.ReplaceAll(path, "/", "")
+
+						return git.LFSObjectIsAssociated(ctx, oid)
+					},
+					name: "LFS file",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.Avatars || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Avatars,
+					isAssociated: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						return user.ExistUserWithAvatar(ctx, path)
+					},
+					name: "avatar",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.RepoAvatars || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.RepoAvatars,
+					isAssociated: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						return repo.ExistRepoWithAvatar(ctx, path)
+					},
+					name: "repo avatar",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.RepoArchives || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.RepoAvatars,
+					isAssociated: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						has, err := repo.ExistsRepoArchiverWithStoragePath(ctx, path)
+						if err == nil || errors.Is(err, util.ErrInvalidArgument) {
+							// invalid arguments mean that the object is not a valid repo archiver and it should be removed
+							return has, nil
+						}
+						return has, err
+					},
+					name: "repo archive",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.Packages || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Packages,
+					isAssociated: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						parts := strings.SplitN(path, "/", 3)
+						if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 || len(parts[2]) < 4 || parts[0]+parts[1] != parts[2][0:4] {
+							return false, nil
+						}
+
+						return packages.ExistPackageBlobWithSHA(ctx, parts[2])
+					},
+					name: "package blob",
+				}); err != nil {
+				return err
+			}
+		}
+
+		return nil
 	}
-	return checkAttachmentStorageFiles(logger, autofix)
 }
 
 func init() {
 	Register(&Check{
-		Title:                      "Check if there is garbage storage files",
+		Title:                      "Check if there are unassociated storage files",
 		Name:                       "storages",
 		IsDefault:                  false,
-		Run:                        checkStorageFiles,
+		Run:                        checkStorage(&storageCheckOptions{All: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are unassociated attachments in storage",
+		Name:                       "storage-attachments",
+		IsDefault:                  false,
+		Run:                        checkStorage(&storageCheckOptions{Attachments: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are unassociated lfs files in storage",
+		Name:                       "storage-lfs",
+		IsDefault:                  false,
+		Run:                        checkStorage(&storageCheckOptions{LFS: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are unassociated avatars in storage",
+		Name:                       "storage-avatars",
+		IsDefault:                  false,
+		Run:                        checkStorage(&storageCheckOptions{Avatars: true, RepoAvatars: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are unassociated archives in storage",
+		Name:                       "storage-archives",
+		IsDefault:                  false,
+		Run:                        checkStorage(&storageCheckOptions{RepoArchives: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are unassociated package blobs in storage",
+		Name:                       "storage-packages",
+		IsDefault:                  false,
+		Run:                        checkStorage(&storageCheckOptions{Packages: true}),
 		AbortIfFailed:              false,
 		SkipDatabaseInitialization: false,
 		Priority:                   1,

modules/git/repo_archive.go

@@ -38,6 +38,18 @@ func (a ArchiveType) String() string {
 	return "unknown"
 }
 
+func ToArchiveType(s string) ArchiveType {
+	switch s {
+	case "zip":
+		return ZIP
+	case "tar.gz":
+		return TARGZ
+	case "bundle":
+		return BUNDLE
+	}
+	return 0
+}
+
 // CreateArchive create archive content to the target path
 func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, target io.Writer, usePrefix bool, commitID string) error {
 	if format.String() == "unknown" {