API add/generalize pagination (#9452)

* paginate results

* fixed deadlock

* prevented breaking change

* updated swagger

* go fmt

* fixed find topic

* go mod tidy

* go mod vendor with go1.13.5

* fixed repo find topics

* fixed unit test

* added Limit method to Engine struct; use engine variable when provided; fixed gitignore

* use ItemsPerPage for default pagesize; fix GetWatchers, getOrgUsersByOrgID and GetStargazers; fix GetAllCommits headers; reverted some changed behaviors

* set Page value on Home route

* improved memory allocations

* fixed response headers

* removed logfiles

* fixed import order

* import order

* improved swagger

* added function to get models.ListOptions from context

* removed pagesize diff on unit test

* fixed imports

* removed unnecessary struct field

* fixed go fmt

* scoped PR

* code improvements

* code improvements

* go mod tidy

* fixed import order

* fixed commit statuses session

* fixed files headers

* fixed headers; added pagination for notifications

* go mod tidy

* go fmt

* removed Private from user search options; added setting.UI.IssuePagingNum as default valeu on repo's issues list

* Apply suggestions from code review

Co-Authored-By: 6543 <6543@obermui.de>
Co-Authored-By: zeripath <art27@cantab.net>

* fixed build error

* CI.restart()

* fixed merge conflicts resolve

* fixed conflicts resolve

* improved FindTrackedTimesOptions.ToOptions() method

* added backwards compatibility on ListReleases request; fixed issue tracked time ToSession

* fixed build error; fixed swagger template

* fixed swagger template

* fixed ListReleases backwards compatibility

* added page to user search route

Co-authored-by: techknowlogick <matti@mdranta.net>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: zeripath <art27@cantab.net>

cmd/admin.go

@@ -351,9 +351,11 @@ func runRepoSyncReleases(c *cli.Context) error {
 	log.Trace("Synchronizing repository releases (this may take a while)")
 	for page := 1; ; page++ {
 		repos, count, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
-			Page:     page,
-			PageSize: models.RepositoryListDefaultPageSize,
-			Private:  true,
+			ListOptions: models.ListOptions{
+				PageSize: models.RepositoryListDefaultPageSize,
+				Page:     page,
+			},
+			Private: true,
 		})
 		if err != nil {
 			return fmt.Errorf("SearchRepositoryByName: %v", err)

models/access.go

@@ -215,7 +215,7 @@ func (repo *Repository) refreshAccesses(e Engine, accessMap map[int64]*userAcces
 
 // refreshCollaboratorAccesses retrieves repository collaborations with their access modes.
 func (repo *Repository) refreshCollaboratorAccesses(e Engine, accessMap map[int64]*userAccess) error {
-	collaborators, err := repo.getCollaborators(e)
+	collaborators, err := repo.getCollaborators(e, ListOptions{})
 	if err != nil {
 		return fmt.Errorf("getCollaborations: %v", err)
 	}

models/commit_status.go

@@ -104,7 +104,7 @@ func CalcCommitStatus(statuses []*CommitStatus) *CommitStatus {
 
 // CommitStatusOptions holds the options for query commit statuses
 type CommitStatusOptions struct {
-	Page     int
+	ListOptions
 	State    string
 	SortType string
 }
@@ -114,18 +114,22 @@ func GetCommitStatuses(repo *Repository, sha string, opts *CommitStatusOptions)
 	if opts.Page <= 0 {
 		opts.Page = 1
 	}
+	if opts.PageSize <= 0 {
+		opts.Page = ItemsPerPage
+	}
 
 	countSession := listCommitStatusesStatement(repo, sha, opts)
+	countSession = opts.setSessionPagination(countSession)
 	maxResults, err := countSession.Count(new(CommitStatus))
 	if err != nil {
 		log.Error("Count PRs: %v", err)
 		return nil, maxResults, err
 	}
 
-	statuses := make([]*CommitStatus, 0, ItemsPerPage)
+	statuses := make([]*CommitStatus, 0, opts.PageSize)
 	findSession := listCommitStatusesStatement(repo, sha, opts)
+	findSession = opts.setSessionPagination(findSession)
 	sortCommitStatusesSession(findSession, opts.SortType)
-	findSession.Limit(ItemsPerPage, (opts.Page-1)*ItemsPerPage)
 	return statuses, maxResults, findSession.Find(&statuses)
 }
 

models/gpg_key.go

@@ -64,9 +64,14 @@ func (key *GPGKey) AfterLoad(session *xorm.Session) {
 }
 
 // ListGPGKeys returns a list of public keys belongs to given user.
-func ListGPGKeys(uid int64) ([]*GPGKey, error) {
-	keys := make([]*GPGKey, 0, 5)
-	return keys, x.Where("owner_id=? AND primary_key_id=''", uid).Find(&keys)
+func ListGPGKeys(uid int64, listOptions ListOptions) ([]*GPGKey, error) {
+	sess := x.Where("owner_id=? AND primary_key_id=''", uid)
+	if listOptions.Page != 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+
+	keys := make([]*GPGKey, 0, 2)
+	return keys, sess.Find(&keys)
 }
 
 // GetGPGKeyByID returns public key by given ID.
@@ -628,7 +633,7 @@ func ParseCommitWithSignature(c *git.Commit) *CommitVerification {
 
 	// Now try to associate the signature with the committer, if present
 	if committer.ID != 0 {
-		keys, err := ListGPGKeys(committer.ID)
+		keys, err := ListGPGKeys(committer.ID, ListOptions{})
 		if err != nil { //Skipping failed to get gpg keys of user
 			log.Error("ListGPGKeys: %v", err)
 			return &CommitVerification{

models/issue.go

@@ -1103,13 +1103,12 @@ func GetIssuesByIDs(issueIDs []int64) ([]*Issue, error) {
 
 // IssuesOptions represents options of an issue.
 type IssuesOptions struct {
+	ListOptions
 	RepoIDs     []int64 // include all repos if empty
 	AssigneeID  int64
 	PosterID    int64
 	MentionedID int64
 	MilestoneID int64
-	Page        int
-	PageSize    int
 	IsClosed    util.OptionalBool
 	IsPull      util.OptionalBool
 	LabelIDs    []int64

models/issue_comment.go

@@ -777,6 +777,7 @@ func GetCommentByID(id int64) (*Comment, error) {
 
 // FindCommentsOptions describes the conditions to Find comments
 type FindCommentsOptions struct {
+	ListOptions
 	RepoID   int64
 	IssueID  int64
 	ReviewID int64
@@ -814,6 +815,11 @@ func findComments(e Engine, opts FindCommentsOptions) ([]*Comment, error) {
 	if opts.RepoID > 0 {
 		sess.Join("INNER", "issue", "issue.id = comment.issue_id")
 	}
+
+	if opts.Page != 0 {
+		sess = opts.setSessionPagination(sess)
+	}
+
 	return comments, sess.
 		Asc("comment.created_unix").
 		Asc("comment.id").

models/issue_label.go

@@ -1,4 +1,5 @@
 // Copyright 2016 The Gogs Authors. All rights reserved.
+// Copyright 2020 The Gitea Authors.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -298,7 +299,7 @@ func GetLabelsInRepoByIDs(repoID int64, labelIDs []int64) ([]*Label, error) {
 		Find(&labels)
 }
 
-func getLabelsByRepoID(e Engine, repoID int64, sortType string) ([]*Label, error) {
+func getLabelsByRepoID(e Engine, repoID int64, sortType string, listOptions ListOptions) ([]*Label, error) {
 	labels := make([]*Label, 0, 10)
 	sess := e.Where("repo_id = ?", repoID)
 
@@ -313,12 +314,16 @@ func getLabelsByRepoID(e Engine, repoID int64, sortType string) ([]*Label, error
 		sess.Asc("name")
 	}
 
+	if listOptions.Page != 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+
 	return labels, sess.Find(&labels)
 }
 
 // GetLabelsByRepoID returns all labels that belong to given repository by ID.
-func GetLabelsByRepoID(repoID int64, sortType string) ([]*Label, error) {
-	return getLabelsByRepoID(x, repoID, sortType)
+func GetLabelsByRepoID(repoID int64, sortType string, listOptions ListOptions) ([]*Label, error) {
+	return getLabelsByRepoID(x, repoID, sortType, listOptions)
 }
 
 func getLabelsByIssueID(e Engine, issueID int64) ([]*Label, error) {

models/issue_label_test.go

@@ -131,7 +131,7 @@ func TestGetLabelsInRepoByIDs(t *testing.T) {
 func TestGetLabelsByRepoID(t *testing.T) {
 	assert.NoError(t, PrepareTestDatabase())
 	testSuccess := func(repoID int64, sortType string, expectedIssueIDs []int64) {
-		labels, err := GetLabelsByRepoID(repoID, sortType)
+		labels, err := GetLabelsByRepoID(repoID, sortType, ListOptions{})
 		assert.NoError(t, err)
 		assert.Len(t, labels, len(expectedIssueIDs))
 		for i, label := range labels {

models/issue_milestone.go

@@ -221,7 +221,7 @@ func (milestones MilestoneList) getMilestoneIDs() []int64 {
 }
 
 // GetMilestonesByRepoID returns all opened milestones of a repository.
-func GetMilestonesByRepoID(repoID int64, state api.StateType) (MilestoneList, error) {
+func GetMilestonesByRepoID(repoID int64, state api.StateType, listOptions ListOptions) (MilestoneList, error) {
 	sess := x.Where("repo_id = ?", repoID)
 
 	switch state {
@@ -238,7 +238,11 @@ func GetMilestonesByRepoID(repoID int64, state api.StateType) (MilestoneList, er
 		sess = sess.And("is_closed = ?", false)
 	}
 
-	miles := make([]*Milestone, 0, 10)
+	if listOptions.Page != 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+
+	miles := make([]*Milestone, 0, listOptions.PageSize)
 	return miles, sess.Asc("deadline_unix").Asc("id").Find(&miles)
 }
 

models/issue_milestone_test.go

@@ -71,7 +71,7 @@ func TestGetMilestonesByRepoID(t *testing.T) {
 	assert.NoError(t, PrepareTestDatabase())
 	test := func(repoID int64, state api.StateType) {
 		repo := AssertExistsAndLoadBean(t, &Repository{ID: repoID}).(*Repository)
-		milestones, err := GetMilestonesByRepoID(repo.ID, state)
+		milestones, err := GetMilestonesByRepoID(repo.ID, state, ListOptions{})
 		assert.NoError(t, err)
 
 		var n int
@@ -105,7 +105,7 @@ func TestGetMilestonesByRepoID(t *testing.T) {
 	test(3, api.StateClosed)
 	test(3, api.StateAll)
 
-	milestones, err := GetMilestonesByRepoID(NonexistentID, api.StateOpen)
+	milestones, err := GetMilestonesByRepoID(NonexistentID, api.StateOpen, ListOptions{})
 	assert.NoError(t, err)
 	assert.Len(t, milestones, 0)
 }

models/issue_reaction.go

@@ -30,6 +30,7 @@ type Reaction struct {
 
 // FindReactionsOptions describes the conditions to Find reactions
 type FindReactionsOptions struct {
+	ListOptions
 	IssueID   int64
 	CommentID int64
 	UserID    int64
@@ -71,20 +72,28 @@ func FindCommentReactions(comment *Comment) (ReactionList, error) {
 }
 
 // FindIssueReactions returns a ReactionList of all reactions from an issue
-func FindIssueReactions(issue *Issue) (ReactionList, error) {
+func FindIssueReactions(issue *Issue, listOptions ListOptions) (ReactionList, error) {
 	return findReactions(x, FindReactionsOptions{
-		IssueID:   issue.ID,
-		CommentID: -1,
+		ListOptions: listOptions,
+		IssueID:     issue.ID,
+		CommentID:   -1,
 	})
 }
 
 func findReactions(e Engine, opts FindReactionsOptions) ([]*Reaction, error) {
-	reactions := make([]*Reaction, 0, 10)
-	sess := e.Where(opts.toConds())
-	return reactions, sess.
+	e = e.
+		Where(opts.toConds()).
 		In("reaction.`type`", setting.UI.Reactions).
-		Asc("reaction.issue_id", "reaction.comment_id", "reaction.created_unix", "reaction.id").
-		Find(&reactions)
+		Asc("reaction.issue_id", "reaction.comment_id", "reaction.created_unix", "reaction.id")
+	if opts.Page != 0 {
+		e = opts.setEnginePagination(e)
+
+		reactions := make([]*Reaction, 0, opts.PageSize)
+		return reactions, e.Find(&reactions)
+	}
+
+	reactions := make([]*Reaction, 0, 10)
+	return reactions, e.Find(&reactions)
 }
 
 func createReaction(e *xorm.Session, opts *ReactionOptions) (*Reaction, error) {

models/issue_stopwatch.go

@@ -33,9 +33,14 @@ func getStopwatch(e Engine, userID, issueID int64) (sw *Stopwatch, exists bool,
 }
 
 // GetUserStopwatches return list of all stopwatches of a user
-func GetUserStopwatches(userID int64) (sws *Stopwatches, err error) {
-	sws = new(Stopwatches)
-	err = x.Where("stopwatch.user_id = ?", userID).Find(sws)
+func GetUserStopwatches(userID int64, listOptions ListOptions) (*Stopwatches, error) {
+	sws := new(Stopwatches)
+	sess := x.Where("stopwatch.user_id = ?", userID)
+	if listOptions.Page != 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+
+	err := sess.Find(sws)
 	if err != nil {
 		return nil, err
 	}

models/issue_test.go

@@ -139,24 +139,30 @@ func TestIssues(t *testing.T) {
 			IssuesOptions{
 				RepoIDs:  []int64{1, 3},
 				SortType: "oldest",
-				Page:     1,
-				PageSize: 4,
+				ListOptions: ListOptions{
+					Page:     1,
+					PageSize: 4,
+				},
 			},
 			[]int64{1, 2, 3, 5},
 		},
 		{
 			IssuesOptions{
 				LabelIDs: []int64{1},
-				Page:     1,
-				PageSize: 4,
+				ListOptions: ListOptions{
+					Page:     1,
+					PageSize: 4,
+				},
 			},
 			[]int64{2, 1},
 		},
 		{
 			IssuesOptions{
 				LabelIDs: []int64{1, 2},
-				Page:     1,
-				PageSize: 4,
+				ListOptions: ListOptions{
+					Page:     1,
+					PageSize: 4,
+				},
 			},
 			[]int64{}, // issues with **both** label 1 and 2, none of these issues matches, TODO: add more tests
 		},

models/issue_tracked_time.go

@@ -10,7 +10,6 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 
 	"xorm.io/builder"
-	"xorm.io/xorm"
 )
 
 // TrackedTime represents a time that was spent for a specific issue.
@@ -71,6 +70,7 @@ func (tl TrackedTimeList) LoadAttributes() (err error) {
 
 // FindTrackedTimesOptions represent the filters for tracked times. If an ID is 0 it will be ignored.
 type FindTrackedTimesOptions struct {
+	ListOptions
 	IssueID           int64
 	UserID            int64
 	RepositoryID      int64
@@ -104,11 +104,19 @@ func (opts *FindTrackedTimesOptions) ToCond() builder.Cond {
 }
 
 // ToSession will convert the given options to a xorm Session by using the conditions from ToCond and joining with issue table if required
-func (opts *FindTrackedTimesOptions) ToSession(e Engine) *xorm.Session {
+func (opts *FindTrackedTimesOptions) ToSession(e Engine) Engine {
+	sess := e
 	if opts.RepositoryID > 0 || opts.MilestoneID > 0 {
-		return e.Join("INNER", "issue", "issue.id = tracked_time.issue_id").Where(opts.ToCond())
+		sess = e.Join("INNER", "issue", "issue.id = tracked_time.issue_id")
 	}
-	return e.Where(opts.ToCond())
+
+	sess = sess.Where(opts.ToCond())
+
+	if opts.Page != 0 {
+		sess = opts.setEnginePagination(sess)
+	}
+
+	return sess
 }
 
 func getTrackedTimes(e Engine, options FindTrackedTimesOptions) (trackedTimes TrackedTimeList, err error) {

models/issue_watch.go

@@ -77,18 +77,22 @@ func GetIssueWatchersIDs(issueID int64) ([]int64, error) {
 }
 
 // GetIssueWatchers returns watchers/unwatchers of a given issue
-func GetIssueWatchers(issueID int64) (IssueWatchList, error) {
-	return getIssueWatchers(x, issueID)
+func GetIssueWatchers(issueID int64, listOptions ListOptions) (IssueWatchList, error) {
+	return getIssueWatchers(x, issueID, listOptions)
 }
 
-func getIssueWatchers(e Engine, issueID int64) (watches IssueWatchList, err error) {
-	err = e.
+func getIssueWatchers(e Engine, issueID int64, listOptions ListOptions) (watches IssueWatchList, err error) {
+	sess := e.
 		Where("`issue_watch`.issue_id = ?", issueID).
 		And("`issue_watch`.is_watching = ?", true).
 		And("`user`.is_active = ?", true).
 		And("`user`.prohibit_login = ?", false).
-		Join("INNER", "`user`", "`user`.id = `issue_watch`.user_id").
-		Find(&watches)
+		Join("INNER", "`user`", "`user`.id = `issue_watch`.user_id")
+
+	if listOptions.Page == 0 {
+		sess = listOptions.setSessionPagination(sess)
+	}
+	err = sess.Find(&watches)
 	return
 }
 

models/issue_watch_test.go

@@ -42,22 +42,22 @@ func TestGetIssueWatch(t *testing.T) {
 func TestGetIssueWatchers(t *testing.T) {
 	assert.NoError(t, PrepareTestDatabase())
 
-	iws, err := GetIssueWatchers(1)
+	iws, err := GetIssueWatchers(1, ListOptions{})
 	assert.NoError(t, err)
 	// Watcher is inactive, thus 0
 	assert.Len(t, iws, 0)
 
-	iws, err = GetIssueWatchers(2)
+	iws, err = GetIssueWatchers(2, ListOptions{})
 	assert.NoError(t, err)
 	// Watcher is explicit not watching
 	assert.Len(t, iws, 0)
 
-	iws, err = GetIssueWatchers(5)
+	iws, err = GetIssueWatchers(5, ListOptions{})
 	assert.NoError(t, err)
 	// Issue has no Watchers
 	assert.Len(t, iws, 0)
 
-	iws, err = GetIssueWatchers(7)
+	iws, err = GetIssueWatchers(7, ListOptions{})
 	assert.NoError(t, err)
 	// Issue has one watcher
 	assert.Len(t, iws, 1)