From 4e0e1078a016363c5909461b123a3923612e58ae Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Mar 2021 16:15:37 +0100 Subject: [PATCH 01/54] Added push mirror model. --- models/repo_pushmirror.go | 112 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 models/repo_pushmirror.go diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go new file mode 100644 index 000000000000..c4e0f00e8568 --- /dev/null +++ b/models/repo_pushmirror.go @@ -0,0 +1,112 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +var ( + // ErrPushMirrorNotExist mirror does not exist error + ErrPushMirrorNotExist = errors.New("PushMirror does not exist") +) + +// PushMirror represents mirror information of a repository. +type PushMirror struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` + Repo *Repository `xorm:"-"` + RemoteName string + + Interval time.Duration + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` + NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` + LastError string +} + +// BeforeInsert will be invoked by XORM before inserting a record +func (m *PushMirror) BeforeInsert() { + if m != nil { + m.UpdatedUnix = timeutil.TimeStampNow() + m.NextUpdateUnix = timeutil.TimeStampNow() + } +} + +// AfterLoad is invoked from XORM after setting the values of all fields of this object. +func (m *PushMirror) AfterLoad(session *xorm.Session) { + if m == nil { + return + } + + var err error + m.Repo, err = getRepositoryByID(session, m.RepoID) + if err != nil { + log.Error("getRepositoryByID[%d]: %v", m.ID, err) + } +} + +// InsertPushMirror inserts a push-mirror to database +func InsertPushMirror(m *PushMirror) error { + _, err := x.Insert(m) + return err +} + +// UpdatePushMirror updates the push-mirror +func UpdatePushMirror(m *PushMirror) error { + _, err := x.ID(m.ID).AllCols().Update(m) + return err +} + +// DeletePushMirrorByID deletes a push-mirrors by ID +func DeletePushMirrorByID(ID int64) error { + _, err := x.Delete(&PushMirror{ID: ID}) + return err +} + +// DeletePushMirrorsByRepoID deletes all push-mirrors by repoID +func DeletePushMirrorsByRepoID(repoID int64) error { + _, err := x.Delete(&PushMirror{RepoID: repoID}) + return err +} + +// GetPushMirrorByID returns push-mirror information. +func GetPushMirrorByID(ID int64) (*PushMirror, error) { + m := &PushMirror{ID: ID} + has, err := e.Get(m) + if err != nil { + return nil, err + } else if !has { + return nil, ErrPushMirrorNotExist + } + return m, nil +} + +// GetPushMirrorsByRepoID returns push-mirror informations of a repository. +func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { + mirrors := make([]*PushMirror, 0, 10) + return mirrors, x.Where("repo_id=?", repoID).Find(&mirrors) +} + +// PushMirrorsIterate iterates all push-mirror repositories. +func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { + return x + .Where("next_update_unix<=?", time.Now().Unix()) + .And("next_update_unix!=0") + .Iterate(new(PushMirror), f) +} + +// ScheduleNextUpdate calculates and sets next update time. +func (m *PushMirror) ScheduleNextUpdate() { + if m.Interval != 0 { + m.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(m.Interval) + } else { + m.NextUpdateUnix = 0 + } +} From 5797f963f8dd265d7e6807272506dc0189babbe5 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 24 Mar 2021 15:22:17 +0000 Subject: [PATCH 02/54] Integrated push mirror into queue. --- services/mirror/mirror.go | 48 ++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index e4981b8c00e6..858b594858ed 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -347,21 +347,41 @@ func Password(m *models.Mirror) string { // Update checks and updates mirror repositories. func Update(ctx context.Context) error { log.Trace("Doing: Update") - if err := models.MirrorsIterate(func(idx int, bean interface{}) error { - m := bean.(*models.Mirror) - if m.Repo == nil { - log.Error("Disconnected mirror repository found: %d", m.ID) + + handler := func(idx int, bean interface{}) error { + var item string + if m, ok := bean.(*models.Mirror); ok { + if m.Repo == nil { + log.Error("Disconnected mirror found: %d", m.ID) + return nil + } + item = fmt.Sprintf("pull %d", m.RepoID) + } else if m, ok := bean.(*models.PushMirror); ok { + if m.Repo == nil { + log.Error("Disconnected push-mirror found: %d", m.ID) + return nil + } + item = fmt.Sprintf("push %d", m.ID) + } else { + log.Error("Unknown bean: %v", bean) return nil } + select { case <-ctx.Done(): return fmt.Errorf("Aborted") default: - mirrorQueue.Add(m.RepoID) + mirrorQueue.Add(item) return nil } - }); err != nil { - log.Trace("Update: %v", err) + } + + if err := models.MirrorsIterate(handler); err != nil { + log.Trace("MirrorsIterate: %v", err) + return err + } + if err := models.PushMirrorsIterate(handler); err != nil { + log.Trace("PushMirrorsIterate: %v", err) return err } log.Trace("Finished: Update") @@ -377,8 +397,15 @@ func SyncMirrors(ctx context.Context) { case <-ctx.Done(): mirrorQueue.Close() return - case repoID := <-mirrorQueue.Queue(): - syncMirror(repoID) + case item := <-mirrorQueue.Queue(): + if strings.HasPrefix(item, "pull") { + syncMirror(item[5:]) + } else if strings.HasPrefix(item, "push") { + //syncPushMirror(item[5:]) + } else { + log.Error("Unknown item in queue: %v", item) + } + mirrorQueue.Remove(item) } } } @@ -393,7 +420,6 @@ func syncMirror(repoID string) { // There was a panic whilst syncMirrors... log.Error("PANIC whilst syncMirrors[%s] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) }() - mirrorQueue.Remove(repoID) id, _ := strconv.ParseInt(repoID, 10, 64) m, err := models.GetMirrorByRepoID(id) @@ -586,5 +612,5 @@ func InitSyncMirrors() { // StartToMirror adds repoID to mirror queue func StartToMirror(repoID int64) { - go mirrorQueue.Add(repoID) + go mirrorQueue.Add(fmt.Sprintf("pull %d", repoID)) } From 00bf2646ff08c82a5703fe3baa1c03b5a9f2926c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 25 Mar 2021 14:43:43 +0000 Subject: [PATCH 03/54] Moved methods into own file. --- models/repo_pushmirror.go | 19 +- services/mirror/mirror.go | 402 +------------------------------- services/mirror/mirror_pull.go | 410 +++++++++++++++++++++++++++++++++ 3 files changed, 424 insertions(+), 407 deletions(-) create mode 100644 services/mirror/mirror_pull.go diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index c4e0f00e8568..e694f590e1d6 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -5,6 +5,7 @@ package models import ( + "errors" "time" "code.gitea.io/gitea/modules/log" @@ -20,10 +21,10 @@ var ( // PushMirror represents mirror information of a repository. type PushMirror struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"INDEX"` - Repo *Repository `xorm:"-"` - RemoteName string + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` + Repo *Repository `xorm:"-"` + RemoteName string Interval time.Duration UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` @@ -79,7 +80,7 @@ func DeletePushMirrorsByRepoID(repoID int64) error { // GetPushMirrorByID returns push-mirror information. func GetPushMirrorByID(ID int64) (*PushMirror, error) { m := &PushMirror{ID: ID} - has, err := e.Get(m) + has, err := x.Get(m) if err != nil { return nil, err } else if !has { @@ -96,10 +97,10 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { // PushMirrorsIterate iterates all push-mirror repositories. func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { - return x - .Where("next_update_unix<=?", time.Now().Unix()) - .And("next_update_unix!=0") - .Iterate(new(PushMirror), f) + return x. + Where("next_update_unix<=?", time.Now().Unix()). + And("next_update_unix!=0"). + Iterate(new(PushMirror), f) } // ScheduleNextUpdate calculates and sets next update time. diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 858b594858ed..a4979d595aa4 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -8,20 +8,15 @@ import ( "context" "fmt" "net/url" - "strconv" "strings" - "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" - "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" ) @@ -121,200 +116,6 @@ func UpdateAddress(m *models.Mirror, addr string) error { return models.UpdateRepositoryCols(m.Repo, "original_url") } -// gitShortEmptySha Git short empty SHA -const gitShortEmptySha = "0000000" - -// mirrorSyncResult contains information of a updated reference. -// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. -// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. -type mirrorSyncResult struct { - refName string - oldCommitID string - newCommitID string -} - -// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. -func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { - results := make([]*mirrorSyncResult, 0, 3) - lines := strings.Split(output, "\n") - for i := range lines { - // Make sure reference name is presented before continue - idx := strings.Index(lines[i], "-> ") - if idx == -1 { - continue - } - - refName := lines[i][idx+3:] - - switch { - case strings.HasPrefix(lines[i], " * "): // New reference - if strings.HasPrefix(lines[i], " * [new tag]") { - refName = git.TagPrefix + refName - } else if strings.HasPrefix(lines[i], " * [new branch]") { - refName = git.BranchPrefix + refName - } - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: gitShortEmptySha, - }) - case strings.HasPrefix(lines[i], " - "): // Delete reference - results = append(results, &mirrorSyncResult{ - refName: refName, - newCommitID: gitShortEmptySha, - }) - case strings.HasPrefix(lines[i], " + "): // Force update - if idx := strings.Index(refName, " "); idx > -1 { - refName = refName[:idx] - } - delimIdx := strings.Index(lines[i][3:], " ") - if delimIdx == -1 { - log.Error("SHA delimiter not found: %q", lines[i]) - continue - } - shas := strings.Split(lines[i][3:delimIdx+3], "...") - if len(shas) != 2 { - log.Error("Expect two SHAs but not what found: %q", lines[i]) - continue - } - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: shas[0], - newCommitID: shas[1], - }) - case strings.HasPrefix(lines[i], " "): // New commits of a reference - delimIdx := strings.Index(lines[i][3:], " ") - if delimIdx == -1 { - log.Error("SHA delimiter not found: %q", lines[i]) - continue - } - shas := strings.Split(lines[i][3:delimIdx+3], "..") - if len(shas) != 2 { - log.Error("Expect two SHAs but not what found: %q", lines[i]) - continue - } - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: shas[0], - newCommitID: shas[1], - }) - - default: - log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) - } - } - return results -} - -// runSync returns true if sync finished without error. -func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { - repoPath := m.Repo.RepoPath() - wikiPath := m.Repo.WikiPath() - timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second - - log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) - gitArgs := []string{"remote", "update"} - if m.EnablePrune { - gitArgs = append(gitArgs, "--prune") - } - - stdoutBuilder := strings.Builder{} - stderrBuilder := strings.Builder{} - if err := git.NewCommand(gitArgs...). - SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). - RunInDirTimeoutPipeline(timeout, repoPath, &stdoutBuilder, &stderrBuilder); err != nil { - stdout := stdoutBuilder.String() - stderr := stderrBuilder.String() - // sanitize the output, since it may contain the remote address, which may - // contain a password - stderrMessage, sanitizeErr := sanitizeOutput(stderr, repoPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) - log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderr, err) - return nil, false - } - stdoutMessage, err := sanitizeOutput(stdout, repoPath) - if err != nil { - log.Error("sanitizeOutput failed: %v", sanitizeErr) - log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderrMessage, err) - return nil, false - } - - log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) - desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, stderrMessage) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return nil, false - } - output := stderrBuilder.String() - - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - log.Error("OpenRepository: %v", err) - return nil, false - } - - log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) - if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { - gitRepo.Close() - log.Error("Failed to synchronize tags to releases for repository: %v", err) - } - gitRepo.Close() - - log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) - if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { - log.Error("Failed to update size for mirror repository: %v", err) - } - - if m.Repo.HasWiki() { - log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) - stderrBuilder.Reset() - stdoutBuilder.Reset() - if err := git.NewCommand("remote", "update", "--prune"). - SetDescription(fmt.Sprintf("Mirror.runSync Wiki: %s ", m.Repo.FullName())). - RunInDirTimeoutPipeline(timeout, wikiPath, &stdoutBuilder, &stderrBuilder); err != nil { - stdout := stdoutBuilder.String() - stderr := stderrBuilder.String() - // sanitize the output, since it may contain the remote address, which may - // contain a password - stderrMessage, sanitizeErr := sanitizeOutput(stderr, repoPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) - log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderr, err) - return nil, false - } - stdoutMessage, err := sanitizeOutput(stdout, repoPath) - if err != nil { - log.Error("sanitizeOutput failed: %v", sanitizeErr) - log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderrMessage, err) - return nil, false - } - - log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) - desc := fmt.Sprintf("Failed to update mirror repository wiki '%s': %s", wikiPath, stderrMessage) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return nil, false - } - log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) - } - - log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) - branches, _, err := repo_module.GetBranches(m.Repo, 0, 0) - if err != nil { - log.Error("GetBranches: %v", err) - return nil, false - } - - for _, branch := range branches { - cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true)) - } - - m.UpdatedUnix = timeutil.TimeStampNow() - return parseRemoteUpdateOutput(output), true -} - // Address returns mirror address from Git repository config without credentials. func Address(m *models.Mirror) string { readAddress(m) @@ -388,9 +189,9 @@ func Update(ctx context.Context) error { return nil } -// SyncMirrors checks and syncs mirrors. +// syncMirrors checks and syncs mirrors. // FIXME: graceful: this should be a persistable queue -func SyncMirrors(ctx context.Context) { +func syncMirrors(ctx context.Context) { // Start listening on new sync requests. for { select { @@ -399,7 +200,7 @@ func SyncMirrors(ctx context.Context) { return case item := <-mirrorQueue.Queue(): if strings.HasPrefix(item, "pull") { - syncMirror(item[5:]) + syncPullMirror(item[5:]) } else if strings.HasPrefix(item, "push") { //syncPushMirror(item[5:]) } else { @@ -410,204 +211,9 @@ func SyncMirrors(ctx context.Context) { } } -func syncMirror(repoID string) { - log.Trace("SyncMirrors [repo_id: %v]", repoID) - defer func() { - err := recover() - if err == nil { - return - } - // There was a panic whilst syncMirrors... - log.Error("PANIC whilst syncMirrors[%s] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) - }() - - id, _ := strconv.ParseInt(repoID, 10, 64) - m, err := models.GetMirrorByRepoID(id) - if err != nil { - log.Error("GetMirrorByRepoID [%s]: %v", repoID, err) - return - } - - log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) - results, ok := runSync(m) - if !ok { - return - } - - log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) - m.ScheduleNextUpdate() - if err = models.UpdateMirror(m); err != nil { - log.Error("UpdateMirror [%s]: %v", repoID, err) - return - } - - var gitRepo *git.Repository - if len(results) == 0 { - log.Trace("SyncMirrors [repo: %-v]: no branches updated", m.Repo) - } else { - log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) - gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) - if err != nil { - log.Error("OpenRepository [%d]: %v", m.RepoID, err) - return - } - defer gitRepo.Close() - - if ok := checkAndUpdateEmptyRepository(m, gitRepo, results); !ok { - return - } - } - - for _, result := range results { - // Discard GitHub pull requests, i.e. refs/pull/* - if strings.HasPrefix(result.refName, "refs/pull/") { - continue - } - - tp, _ := git.SplitRefName(result.refName) - - // Create reference - if result.oldCommitID == gitShortEmptySha { - if tp == git.TagPrefix { - tp = "tag" - } else if tp == git.BranchPrefix { - tp = "branch" - } - commitID, err := gitRepo.GetRefCommitID(result.refName) - if err != nil { - log.Error("gitRepo.GetRefCommitID [repo_id: %s, ref_name: %s]: %v", m.RepoID, result.refName, err) - continue - } - notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ - RefFullName: result.refName, - OldCommitID: git.EmptySHA, - NewCommitID: commitID, - }, repo_module.NewPushCommits()) - notification.NotifySyncCreateRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) - continue - } - - // Delete reference - if result.newCommitID == gitShortEmptySha { - notification.NotifySyncDeleteRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) - continue - } - - // Push commits - oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) - if err != nil { - log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) - continue - } - newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) - if err != nil { - log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) - continue - } - commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) - if err != nil { - log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) - continue - } - - theCommits := repo_module.ListToPushCommits(commits) - if len(theCommits.Commits) > setting.UI.FeedMaxCommitNum { - theCommits.Commits = theCommits.Commits[:setting.UI.FeedMaxCommitNum] - } - - theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID) - - notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ - RefFullName: result.refName, - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - }, theCommits) - } - log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) - - // Get latest commit date and update to current repository updated time - commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) - if err != nil { - log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err) - return - } - - if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil { - log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) - return - } - - log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) -} - -func checkAndUpdateEmptyRepository(m *models.Mirror, gitRepo *git.Repository, results []*mirrorSyncResult) bool { - if !m.Repo.IsEmpty { - return true - } - - hasDefault := false - hasMaster := false - hasMain := false - defaultBranchName := m.Repo.DefaultBranch - if len(defaultBranchName) == 0 { - defaultBranchName = setting.Repository.DefaultBranch - } - firstName := "" - for _, result := range results { - if strings.HasPrefix(result.refName, "refs/pull/") { - continue - } - tp, name := git.SplitRefName(result.refName) - if len(tp) > 0 && tp != git.BranchPrefix { - continue - } - if len(firstName) == 0 { - firstName = name - } - - hasDefault = hasDefault || name == defaultBranchName - hasMaster = hasMaster || name == "master" - hasMain = hasMain || name == "main" - } - - if len(firstName) > 0 { - if hasDefault { - m.Repo.DefaultBranch = defaultBranchName - } else if hasMaster { - m.Repo.DefaultBranch = "master" - } else if hasMain { - m.Repo.DefaultBranch = "main" - } else { - m.Repo.DefaultBranch = firstName - } - // Update the git repository default branch - if err := gitRepo.SetDefaultBranch(m.Repo.DefaultBranch); err != nil { - if !git.IsErrUnsupportedVersion(err) { - log.Error("Failed to update default branch of underlying git repository %-v. Error: %v", m.Repo, err) - desc := fmt.Sprintf("Failed to uupdate default branch of underlying git repository '%s': %v", m.Repo.RepoPath(), err) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return false - } - } - m.Repo.IsEmpty = false - // Update the is empty and default_branch columns - if err := models.UpdateRepositoryCols(m.Repo, "default_branch", "is_empty"); err != nil { - log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err) - desc := fmt.Sprintf("Failed to uupdate default branch of repository '%s': %v", m.Repo.RepoPath(), err) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return false - } - } - return true -} - // InitSyncMirrors initializes a go routine to sync the mirrors func InitSyncMirrors() { - go graceful.GetManager().RunWithShutdownContext(SyncMirrors) + go graceful.GetManager().RunWithShutdownContext(syncMirrors) } // StartToMirror adds repoID to mirror queue diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go new file mode 100644 index 000000000000..b65258f9ef19 --- /dev/null +++ b/services/mirror/mirror_pull.go @@ -0,0 +1,410 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package mirror + +import ( + "fmt" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" +) + +// gitShortEmptySha Git short empty SHA +const gitShortEmptySha = "0000000" + +// mirrorSyncResult contains information of a updated reference. +// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. +// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. +type mirrorSyncResult struct { + refName string + oldCommitID string + newCommitID string +} + +// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. +func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { + results := make([]*mirrorSyncResult, 0, 3) + lines := strings.Split(output, "\n") + for i := range lines { + // Make sure reference name is presented before continue + idx := strings.Index(lines[i], "-> ") + if idx == -1 { + continue + } + + refName := lines[i][idx+3:] + + switch { + case strings.HasPrefix(lines[i], " * "): // New reference + if strings.HasPrefix(lines[i], " * [new tag]") { + refName = git.TagPrefix + refName + } else if strings.HasPrefix(lines[i], " * [new branch]") { + refName = git.BranchPrefix + refName + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " - "): // Delete reference + results = append(results, &mirrorSyncResult{ + refName: refName, + newCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " + "): // Force update + if idx := strings.Index(refName, " "); idx > -1 { + refName = refName[:idx] + } + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error("SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "...") + if len(shas) != 2 { + log.Error("Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + case strings.HasPrefix(lines[i], " "): // New commits of a reference + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error("SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "..") + if len(shas) != 2 { + log.Error("Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + + default: + log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) + } + } + return results +} + +// runSync returns true if sync finished without error. +func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { + repoPath := m.Repo.RepoPath() + wikiPath := m.Repo.WikiPath() + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) + gitArgs := []string{"remote", "update"} + if m.EnablePrune { + gitArgs = append(gitArgs, "--prune") + } + + stdoutBuilder := strings.Builder{} + stderrBuilder := strings.Builder{} + if err := git.NewCommand(gitArgs...). + SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). + RunInDirTimeoutPipeline(timeout, repoPath, &stdoutBuilder, &stderrBuilder); err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() + // sanitize the output, since it may contain the remote address, which may + // contain a password + stderrMessage, sanitizeErr := sanitizeOutput(stderr, repoPath) + if sanitizeErr != nil { + log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) + log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderr, err) + return nil, false + } + stdoutMessage, err := sanitizeOutput(stdout, repoPath) + if err != nil { + log.Error("sanitizeOutput failed: %v", sanitizeErr) + log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderrMessage, err) + return nil, false + } + + log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, stderrMessage) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return nil, false + } + output := stderrBuilder.String() + + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + log.Error("OpenRepository: %v", err) + return nil, false + } + + log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) + if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { + gitRepo.Close() + log.Error("Failed to synchronize tags to releases for repository: %v", err) + } + gitRepo.Close() + + log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) + if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { + log.Error("Failed to update size for mirror repository: %v", err) + } + + if m.Repo.HasWiki() { + log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) + stderrBuilder.Reset() + stdoutBuilder.Reset() + if err := git.NewCommand("remote", "update", "--prune"). + SetDescription(fmt.Sprintf("Mirror.runSync Wiki: %s ", m.Repo.FullName())). + RunInDirTimeoutPipeline(timeout, wikiPath, &stdoutBuilder, &stderrBuilder); err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() + // sanitize the output, since it may contain the remote address, which may + // contain a password + stderrMessage, sanitizeErr := sanitizeOutput(stderr, repoPath) + if sanitizeErr != nil { + log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) + log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderr, err) + return nil, false + } + stdoutMessage, err := sanitizeOutput(stdout, repoPath) + if err != nil { + log.Error("sanitizeOutput failed: %v", sanitizeErr) + log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderrMessage, err) + return nil, false + } + + log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + desc := fmt.Sprintf("Failed to update mirror repository wiki '%s': %s", wikiPath, stderrMessage) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return nil, false + } + log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) + } + + log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) + branches, _, err := repo_module.GetBranches(m.Repo, 0, 0) + if err != nil { + log.Error("GetBranches: %v", err) + return nil, false + } + + for _, branch := range branches { + cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true)) + } + + m.UpdatedUnix = timeutil.TimeStampNow() + return parseRemoteUpdateOutput(output), true +} + +func syncPullMirror(repoID string) { + log.Trace("SyncMirrors [repo_id: %v]", repoID) + defer func() { + err := recover() + if err == nil { + return + } + // There was a panic whilst syncMirrors... + log.Error("PANIC whilst syncMirrors[%s] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) + }() + + id, _ := strconv.ParseInt(repoID, 10, 64) + m, err := models.GetMirrorByRepoID(id) + if err != nil { + log.Error("GetMirrorByRepoID [%s]: %v", repoID, err) + return + } + + log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) + results, ok := runSync(m) + if !ok { + return + } + + log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) + m.ScheduleNextUpdate() + if err = models.UpdateMirror(m); err != nil { + log.Error("UpdateMirror [%s]: %v", repoID, err) + return + } + + var gitRepo *git.Repository + if len(results) == 0 { + log.Trace("SyncMirrors [repo: %-v]: no branches updated", m.Repo) + } else { + log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) + gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) + if err != nil { + log.Error("OpenRepository [%d]: %v", m.RepoID, err) + return + } + defer gitRepo.Close() + + if ok := checkAndUpdateEmptyRepository(m, gitRepo, results); !ok { + return + } + } + + for _, result := range results { + // Discard GitHub pull requests, i.e. refs/pull/* + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + + tp, _ := git.SplitRefName(result.refName) + + // Create reference + if result.oldCommitID == gitShortEmptySha { + if tp == git.TagPrefix { + tp = "tag" + } else if tp == git.BranchPrefix { + tp = "branch" + } + commitID, err := gitRepo.GetRefCommitID(result.refName) + if err != nil { + log.Error("gitRepo.GetRefCommitID [repo_id: %s, ref_name: %s]: %v", m.RepoID, result.refName, err) + continue + } + notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ + RefFullName: result.refName, + OldCommitID: git.EmptySHA, + NewCommitID: commitID, + }, repo_module.NewPushCommits()) + notification.NotifySyncCreateRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) + continue + } + + // Delete reference + if result.newCommitID == gitShortEmptySha { + notification.NotifySyncDeleteRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) + continue + } + + // Push commits + oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) + if err != nil { + log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) + if err != nil { + log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) + if err != nil { + log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) + continue + } + + theCommits := repo_module.ListToPushCommits(commits) + if len(theCommits.Commits) > setting.UI.FeedMaxCommitNum { + theCommits.Commits = theCommits.Commits[:setting.UI.FeedMaxCommitNum] + } + + theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID) + + notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ + RefFullName: result.refName, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + }, theCommits) + } + log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) + + // Get latest commit date and update to current repository updated time + commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) + if err != nil { + log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err) + return + } + + if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil { + log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) + return + } + + log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) +} + +func checkAndUpdateEmptyRepository(m *models.Mirror, gitRepo *git.Repository, results []*mirrorSyncResult) bool { + if !m.Repo.IsEmpty { + return true + } + + hasDefault := false + hasMaster := false + hasMain := false + defaultBranchName := m.Repo.DefaultBranch + if len(defaultBranchName) == 0 { + defaultBranchName = setting.Repository.DefaultBranch + } + firstName := "" + for _, result := range results { + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + tp, name := git.SplitRefName(result.refName) + if len(tp) > 0 && tp != git.BranchPrefix { + continue + } + if len(firstName) == 0 { + firstName = name + } + + hasDefault = hasDefault || name == defaultBranchName + hasMaster = hasMaster || name == "master" + hasMain = hasMain || name == "main" + } + + if len(firstName) > 0 { + if hasDefault { + m.Repo.DefaultBranch = defaultBranchName + } else if hasMaster { + m.Repo.DefaultBranch = "master" + } else if hasMain { + m.Repo.DefaultBranch = "main" + } else { + m.Repo.DefaultBranch = firstName + } + // Update the git repository default branch + if err := gitRepo.SetDefaultBranch(m.Repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + log.Error("Failed to update default branch of underlying git repository %-v. Error: %v", m.Repo, err) + desc := fmt.Sprintf("Failed to uupdate default branch of underlying git repository '%s': %v", m.Repo.RepoPath(), err) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return false + } + } + m.Repo.IsEmpty = false + // Update the is empty and default_branch columns + if err := models.UpdateRepositoryCols(m.Repo, "default_branch", "is_empty"); err != nil { + log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err) + desc := fmt.Sprintf("Failed to uupdate default branch of repository '%s': %v", m.Repo.RepoPath(), err) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return false + } + } + return true +} From 000cf564d6b5cf1d7f7dfdee9b590b72821b3a85 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 26 Mar 2021 07:05:26 +0000 Subject: [PATCH 04/54] Added basic implementation. --- modules/git/repo.go | 24 +++++++--- services/mirror/mirror.go | 2 +- services/mirror/mirror_push.go | 80 ++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 services/mirror/mirror_push.go diff --git a/modules/git/repo.go b/modules/git/repo.go index 515899ab0498..e06cd439353b 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -182,10 +182,12 @@ func Pull(repoPath string, opts PullRemoteOptions) error { // PushOptions options when push to remote type PushOptions struct { - Remote string - Branch string - Force bool - Env []string + Remote string + Branch string + Force bool + Mirror bool + Env []string + Timeout time.Duration } // Push pushs local commits to given remote branch. @@ -194,10 +196,20 @@ func Push(repoPath string, opts PushOptions) error { if opts.Force { cmd.AddArguments("-f") } - cmd.AddArguments("--", opts.Remote, opts.Branch) + if opts.Mirror { + cmd.AddArguments("--mirror") + } + cmd.AddArguments("--", opts.Remote) + if len(opts.Branch) > 0 { + cmd.AddArguments(opts.Branch) + } var outbuf, errbuf strings.Builder - err := cmd.RunInDirTimeoutEnvPipeline(opts.Env, -1, repoPath, &outbuf, &errbuf) + if opts.Timeout == 0 { + opts.Timeout = -1 + } + + err := cmd.RunInDirTimeoutEnvPipeline(opts.Env, opts.Timeout, repoPath, &outbuf, &errbuf) if err != nil { if strings.Contains(errbuf.String(), "non-fast-forward") { return &ErrPushOutOfDate{ diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index a4979d595aa4..2319d4dc33e4 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -202,7 +202,7 @@ func syncMirrors(ctx context.Context) { if strings.HasPrefix(item, "pull") { syncPullMirror(item[5:]) } else if strings.HasPrefix(item, "push") { - //syncPushMirror(item[5:]) + syncPushMirror(item[5:]) } else { log.Error("Unknown item in queue: %v", item) } diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go new file mode 100644 index 000000000000..5b238976d52b --- /dev/null +++ b/services/mirror/mirror_push.go @@ -0,0 +1,80 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package mirror + +import ( + "fmt" + "strconv" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +func syncPushMirror(mirrorID string) { + log.Trace("SyncPushMirror [mirror_id: %v]", mirrorID) + defer func() { + err := recover() + if err == nil { + return + } + // There was a panic whilst syncPushMirror... + log.Error("PANIC whilst syncPushMirror[%s] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) + }() + + id, _ := strconv.ParseInt(mirrorID, 10, 64) + m, err := models.GetPushMirrorByID(id) + if err != nil { + log.Error("GetPushMirrorByID [%d]: %v", id, err) + return + } + + log.Trace("SyncPushMirror [repo: %-v]: Running Sync", m.Repo) + err = runPushSync(m) + if err != nil { + log.Error("SyncPushMirror [%d]: %v", id, err) + return + } + + log.Trace("SyncPushMirror [repo: %-v]: Scheduling next update", m.Repo) + m.ScheduleNextUpdate() + if err = models.UpdatePushMirror(m); err != nil { + log.Error("UpdatePushMirror [%d]: %v", id, err) + return + } + + log.Trace("SyncPushMirror [repo: %-v]: Successfully updated", m.Repo) +} + +// runPushSync returns true if sync finished without error. +func runPushSync(m *models.PushMirror) error { + repoPath := m.Repo.RepoPath() + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + log.Trace("SyncPushMirror [repo: %-v]: running git push...", m.Repo) + + if err := git.Push(repoPath, git.PushOptions{ + Remote: m.RemoteName, + Force: true, + Mirror: true, + Timeout: timeout, + }); err != nil { + if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { + return err + } + return fmt.Errorf("Push: %v", err) + } + + // TODO LFS + + /* Should the wiki be mirrored too? + if m.Repo.HasWiki() { + + }*/ + + return nil +} From 5e701eb6efa7040e57a36f1117773a413f8fb03d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 27 Mar 2021 10:56:46 +0000 Subject: [PATCH 05/54] Mirror wiki too. --- services/mirror/mirror_push.go | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 5b238976d52b..9bc1f7fe8cb1 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -57,24 +57,34 @@ func runPushSync(m *models.PushMirror) error { log.Trace("SyncPushMirror [repo: %-v]: running git push...", m.Repo) - if err := git.Push(repoPath, git.PushOptions{ - Remote: m.RemoteName, - Force: true, - Mirror: true, - Timeout: timeout, - }); err != nil { - if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { - return err + performPush := func(path string) error { + if err := git.Push(path, git.PushOptions{ + Remote: m.RemoteName, + Force: true, + Mirror: true, + Timeout: timeout, + }); err != nil { + if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { + return err + } + return fmt.Errorf("Error pushing remote %s to %s: %v", m.RemoteName, path, err) } - return fmt.Errorf("Push: %v", err) + return nil + } + + err := performPush(repoPath) + if err != nil { + return err } // TODO LFS - /* Should the wiki be mirrored too? if m.Repo.HasWiki() { - - }*/ + err := performPush(m.Repo.WikiPath()); + if err != nil { + return nil + } + } return nil } From ecbbc677bac9bcc3ef39b3cd86bbc822f4937b8e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 28 Mar 2021 18:10:51 +0000 Subject: [PATCH 06/54] Removed duplicated method. --- modules/templates/helper.go | 1 - services/mirror/mirror.go | 24 ------------------------ services/mirror/mirror_pull.go | 13 +++++++++++++ templates/repo/settings/options.tmpl | 2 +- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 7e33f262094e..726db4ed26d1 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -296,7 +296,6 @@ func NewFuncMap() []template.FuncMap { }, "CommentMustAsDiff": gitdiff.CommentMustAsDiff, "MirrorAddress": mirror_service.Address, - "MirrorFullAddress": mirror_service.AddressNoCredentials, "MirrorUserName": mirror_service.Username, "MirrorPassword": mirror_service.Password, "CommitType": func(commit interface{}) string { diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 2319d4dc33e4..3d96878c3e1e 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -59,30 +59,6 @@ func remoteAddress(repoPath string) (string, error) { return "", nil } -// sanitizeOutput sanitizes output of a command, replacing occurrences of the -// repository's remote address with a sanitized version. -func sanitizeOutput(output, repoPath string) (string, error) { - remoteAddr, err := remoteAddress(repoPath) - if err != nil { - // if we're unable to load the remote address, then we're unable to - // sanitize. - return "", err - } - return util.SanitizeMessage(output, remoteAddr), nil -} - -// AddressNoCredentials returns mirror address from Git repository config without credentials. -func AddressNoCredentials(m *models.Mirror) string { - readAddress(m) - u, err := url.Parse(m.Address) - if err != nil { - // this shouldn't happen but just return it unsanitised - return m.Address - } - u.User = nil - return u.String() -} - // UpdateAddress writes new address to Git repository and database func UpdateAddress(m *models.Mirror, addr string) error { repoPath := m.Repo.RepoPath() diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index b65258f9ef19..882decd142ef 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -18,6 +18,7 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" ) // gitShortEmptySha Git short empty SHA @@ -408,3 +409,15 @@ func checkAndUpdateEmptyRepository(m *models.Mirror, gitRepo *git.Repository, re } return true } + +// sanitizeOutput sanitizes output of a command, replacing occurrences of the +// repository's remote address with a sanitized version. +func sanitizeOutput(output, repoPath string) (string, error) { + remoteAddr, err := readRemoteAddress(repoPath, pullMirrorRemoteName) + if err != nil { + // if we're unable to load the remote address, then we're unable to + // sanitize. + return "", err + } + return util.SanitizeMessage(output, remoteAddr), nil +} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index f944eb8d8dea..0d668684ef02 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -91,7 +91,7 @@
- +

{{.i18n.Tr "repo.mirror_address_desc"}}

From f6f4dba29679bed46f72509a66ce69fc9f363fb0 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 28 Mar 2021 21:20:00 +0200 Subject: [PATCH 07/54] Get url for different remotes. --- services/mirror/mirror.go | 24 ++++++++++++------------ services/mirror/mirror_pull.go | 2 ++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 3d96878c3e1e..12e04e8e6ccc 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -28,22 +28,22 @@ func readAddress(m *models.Mirror) { return } var err error - m.Address, err = remoteAddress(m.Repo.RepoPath()) + m.Address, err = readRemoteAddress(m.Repo.RepoPath(), pullMirrorRemoteName) if err != nil { - log.Error("remoteAddress: %v", err) + log.Error("readRemoteAddress: %v", err) } } -func remoteAddress(repoPath string) (string, error) { - var cmd *git.Command +func readRemoteAddress(repoPath, remoteName string) (string, error) { err := git.LoadGitVersion() if err != nil { return "", err } + var cmd *git.Command if git.CheckGitVersionAtLeast("2.7") == nil { - cmd = git.NewCommand("remote", "get-url", "origin") + cmd = git.NewCommand("remote", "get-url", remoteName) } else { - cmd = git.NewCommand("config", "--get", "remote.origin.url") + cmd = git.NewCommand("config", "--get", "remote."+remoteName+".url") } result, err := cmd.RunInDir(repoPath) @@ -62,13 +62,13 @@ func remoteAddress(repoPath string) (string, error) { // UpdateAddress writes new address to Git repository and database func UpdateAddress(m *models.Mirror, addr string) error { repoPath := m.Repo.RepoPath() - // Remove old origin - _, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath) + // Remove old remote + _, err := git.NewCommand("remote", "rm", pullMirrorRemoteName).RunInDir(repoPath) if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { return err } - _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", addr).RunInDir(repoPath) + _, err = git.NewCommand("remote", "add", pullMirrorRemoteName, "--mirror=fetch", addr).RunInDir(repoPath) if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { return err } @@ -76,13 +76,13 @@ func UpdateAddress(m *models.Mirror, addr string) error { if m.Repo.HasWiki() { wikiPath := m.Repo.WikiPath() wikiRemotePath := repo_module.WikiRemoteURL(addr) - // Remove old origin of wiki - _, err := git.NewCommand("remote", "rm", "origin").RunInDir(wikiPath) + // Remove old remote of wiki + _, err := git.NewCommand("remote", "rm", pullMirrorRemoteName).RunInDir(wikiPath) if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { return err } - _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", wikiRemotePath).RunInDir(wikiPath) + _, err = git.NewCommand("remote", "add", pullMirrorRemoteName, "--mirror=fetch", wikiRemotePath).RunInDir(wikiPath) if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { return err } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 882decd142ef..a7ee5c10e048 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -21,6 +21,8 @@ import ( "code.gitea.io/gitea/modules/util" ) +const pullMirrorRemoteName = "origin" + // gitShortEmptySha Git short empty SHA const gitShortEmptySha = "0000000" From 236576084edb00ce27fc5bafdaa58708a9072b22 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 31 Mar 2021 18:22:26 +0000 Subject: [PATCH 08/54] Added migration. --- models/migrations/migrations.go | 2 ++ models/migrations/v178.go | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 models/migrations/v178.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 2c85ebdfd7af..a54041cc2c7b 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -302,6 +302,8 @@ var migrations = []Migration{ NewMigration("Remove invalid labels from comments", removeInvalidLabels), // v177 -> v178 NewMigration("Delete orphaned IssueLabels", deleteOrphanedIssueLabels), + // v178 -> v179 + NewMigration("Create PushMirror table", createPushMirrorTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v178.go b/models/migrations/v178.go new file mode 100644 index 000000000000..7205750cdf30 --- /dev/null +++ b/models/migrations/v178.go @@ -0,0 +1,39 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "time" + + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func createPushMirrorTable(x *xorm.Engine) error { + type PushMirror struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` + RemoteName string + + Interval time.Duration + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` + NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` + LastError string + } + + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + if err := sess.Sync2(new(PushMirror)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + + return sess.Commit() +} From aff13d6ef80d40c3ea6366c20393f14e60c692a8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 31 Mar 2021 21:15:00 +0200 Subject: [PATCH 09/54] Unified remote url access. --- models/repo_mirror.go | 17 ++++++++++ models/repo_pushmirror.go | 12 ++++++- modules/git/remote.go | 31 +++++++++++++++++ modules/templates/helper.go | 31 ++++++++++++++--- services/mirror/mirror.go | 67 ------------------------------------- templates/repo/header.tmpl | 2 +- 6 files changed, 87 insertions(+), 73 deletions(-) create mode 100644 modules/git/remote.go diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 10b0a7b1396d..249b70679461 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -14,6 +14,12 @@ import ( "xorm.io/xorm" ) +// RemoteMirrorer +type RemoteMirrorer interface { + GetRepository() *Repository + GetRemoteName() string +} + // Mirror represents mirror information of a repository. type Mirror struct { ID int64 `xorm:"pk autoincr"` @@ -49,6 +55,17 @@ func (m *Mirror) AfterLoad(session *xorm.Session) { } } +// GetRepository returns the repository. +func (m *Mirror) GetRepository() *Repository { + return m.Repo +} + +// GetRemoteName returns the name of the remote. +func (m *Mirror) GetRemoteName() string { + return "origin" +} + + // ScheduleNextUpdate calculates and sets next update time. func (m *Mirror) ScheduleNextUpdate() { if m.Interval != 0 { diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index e694f590e1d6..383ebb960cd1 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -35,7 +35,7 @@ type PushMirror struct { // BeforeInsert will be invoked by XORM before inserting a record func (m *PushMirror) BeforeInsert() { if m != nil { - m.UpdatedUnix = timeutil.TimeStampNow() + m.UpdatedUnix = 0 m.NextUpdateUnix = timeutil.TimeStampNow() } } @@ -53,6 +53,16 @@ func (m *PushMirror) AfterLoad(session *xorm.Session) { } } +// GetRepository returns the path of the repository. +func (m *PushMirror) GetRepository() *Repository { + return m.Repo +} + +// GetRemoteName returns the name of the remote. +func (m *PushMirror) GetRemoteName() string { + return m.RemoteName +} + // InsertPushMirror inserts a push-mirror to database func InsertPushMirror(m *PushMirror) error { _, err := x.Insert(m) diff --git a/modules/git/remote.go b/modules/git/remote.go new file mode 100644 index 000000000000..a36065ecfbae --- /dev/null +++ b/modules/git/remote.go @@ -0,0 +1,31 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import "net/url" + +// GetRemoteAddress returns the url of a specific remote of the repository. +func GetRemoteAddress(repoPath, remoteName string) (*url.URL, error) { + err := LoadGitVersion() + if err != nil { + return nil, err + } + var cmd *Command + if CheckGitVersionAtLeast("2.7") == nil { + cmd = NewCommand("remote", "get-url", remoteName) + } else { + cmd = NewCommand("config", "--get", "remote."+remoteName+".url") + } + + result, err := cmd.RunInDir(repoPath) + if err != nil { + return nil, err + } + + if len(result) > 0 { + result = result[:len(result)-1] + } + return url.Parse(result) +} diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 726db4ed26d1..fa1354f4f87b 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -27,6 +27,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/emoji" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/repository" @@ -35,7 +36,6 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/gitdiff" - mirror_service "code.gitea.io/gitea/services/mirror" "github.com/editorconfig/editorconfig-core-go/v2" jsoniter "github.com/json-iterator/go" @@ -295,9 +295,7 @@ func NewFuncMap() []template.FuncMap { return float32(n) * 100 / float32(sum) }, "CommentMustAsDiff": gitdiff.CommentMustAsDiff, - "MirrorAddress": mirror_service.Address, - "MirrorUserName": mirror_service.Username, - "MirrorPassword": mirror_service.Password, + "MirrorRemoteAddress": mirrorRemoteAddress, "CommitType": func(commit interface{}) string { switch commit.(type) { case models.SignCommitWithStatuses: @@ -945,3 +943,28 @@ func buildSubjectBodyTemplate(stpl *texttmpl.Template, btpl *template.Template, log.Warn("Failed to parse template [%s/body]: %v", name, err) } } + +type remoteAddress struct { + Address string + Username string + Password string +} + +func mirrorRemoteAddress(m models.RemoteMirrorer) (remoteAddress) { + a := remoteAddress{} + + u, err := git.GetRemoteAddress(m.GetRepository().RepoPath(), m.GetRemoteName()) + if err != nil { + log.Error("GetRemoteAddress %v", err) + return a + } + + if u.User != nil { + a.Username = u.User.Username() + a.Password, _ = u.User.Password() + } + u.User = nil + a.Address = u.String() + + return a +} \ No newline at end of file diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 12e04e8e6ccc..3c3a57616234 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -7,7 +7,6 @@ package mirror import ( "context" "fmt" - "net/url" "strings" "code.gitea.io/gitea/models" @@ -17,48 +16,11 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" - "code.gitea.io/gitea/modules/util" ) // mirrorQueue holds an UniqueQueue object of the mirror var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) -func readAddress(m *models.Mirror) { - if len(m.Address) > 0 { - return - } - var err error - m.Address, err = readRemoteAddress(m.Repo.RepoPath(), pullMirrorRemoteName) - if err != nil { - log.Error("readRemoteAddress: %v", err) - } -} - -func readRemoteAddress(repoPath, remoteName string) (string, error) { - err := git.LoadGitVersion() - if err != nil { - return "", err - } - var cmd *git.Command - if git.CheckGitVersionAtLeast("2.7") == nil { - cmd = git.NewCommand("remote", "get-url", remoteName) - } else { - cmd = git.NewCommand("config", "--get", "remote."+remoteName+".url") - } - - result, err := cmd.RunInDir(repoPath) - if err != nil { - if strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return "", nil - } - return "", err - } - if len(result) > 0 { - return result[:len(result)-1], nil - } - return "", nil -} - // UpdateAddress writes new address to Git repository and database func UpdateAddress(m *models.Mirror, addr string) error { repoPath := m.Repo.RepoPath() @@ -92,35 +54,6 @@ func UpdateAddress(m *models.Mirror, addr string) error { return models.UpdateRepositoryCols(m.Repo, "original_url") } -// Address returns mirror address from Git repository config without credentials. -func Address(m *models.Mirror) string { - readAddress(m) - return util.SanitizeURLCredentials(m.Address, false) -} - -// Username returns the mirror address username -func Username(m *models.Mirror) string { - readAddress(m) - u, err := url.Parse(m.Address) - if err != nil { - // this shouldn't happen but if it does return "" - return "" - } - return u.User.Username() -} - -// Password returns the mirror address password -func Password(m *models.Mirror) string { - readAddress(m) - u, err := url.Parse(m.Address) - if err != nil { - // this shouldn't happen but if it does return "" - return "" - } - password, _ := u.User.Password() - return password -} - // Update checks and updates mirror repositories. func Update(ctx context.Context) error { log.Trace("Doing: Update") diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 71963d698c71..03d453444c36 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -36,7 +36,7 @@ {{end}}
- {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{if .SanitizedOriginalURL}}{{.SanitizedOriginalURL}}{{else}}{{MirrorAddress $.Mirror}}{{end}}
{{end}} + {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{if .SanitizedOriginalURL}}{{.SanitizedOriginalURL}}{{else}}{{(MirrorRemoteAddress $.Mirror).Address}}{{end}}
{{end}} {{if .IsFork}}
{{$.i18n.Tr "repo.forked_from"}} {{SubStr .BaseRepo.RelLink 1 -1}}
{{end}} {{if .IsGenerated}}
{{$.i18n.Tr "repo.generated_from"}} {{SubStr .TemplateRepo.RelLink 1 -1}}
{{end}} From ff4b4ab3f446600aa27573405b9c52e3e1186ed6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 31 Mar 2021 21:20:00 +0200 Subject: [PATCH 10/54] Add/Remove push mirror remotes. --- services/mirror/mirror_push.go | 37 +++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 9bc1f7fe8cb1..8cb970467e3e 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -12,9 +12,44 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" ) +// AddPushMirrorRemote registers the push mirror remote. +func AddPushMirrorRemote(m *models.PushMirror, addr string) error { + if _, err := git.NewCommand("remote", "add", m.RemoteName, addr).RunInDir(m.Repo.RepoPath()); err != nil { + return err + } + + if repo.HasWiki() { + wikiRemotePath := repository.WikiRemoteURL(addr) + if len(wikiRemotePath) > 0 { + if _, err := git.NewCommand("remote", "add", m.RemoteName, wikiRemotePath).RunInDir(m.Repo.WikiPath()); err != nil { + return err + } + } + } + + return nil +} + +// RemovePushMirrorRemote removes the push mirror remote. +func RemovePushMirrorRemote(m *models.PushMirror) error { + cmd := git.NewCommand("remote", "rm", m.RemoteName) + + if _, err := cmd.RunInDir(m.Repo.RepoPath()); err != nil { + return err + } + + if _, err := cmd.RunInDir(m.Repo.WikiPath()); err != nil { + // The wiki remote may not exist + log.Warning("Wiki Remote[%d] could not be removed: %v", m.ID, err) + } + + return nil +} + func syncPushMirror(mirrorID string) { log.Trace("SyncPushMirror [mirror_id: %v]", mirrorID) defer func() { From 382be0768107129145c1313d8632a3002a9fc5ed Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 31 Mar 2021 21:47:00 +0200 Subject: [PATCH 11/54] Prevent hangs with missing credentials. --- modules/git/command.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/git/command.go b/modules/git/command.go index fe258954628e..f6e6a45cb78e 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -129,6 +129,7 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time. cmd.Env = env cmd.Env = append(cmd.Env, fmt.Sprintf("LC_ALL=%s", DefaultLocale)) } + cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0") // TODO: verify if this is still needed in golang 1.15 if goVersionLessThan115 { From bf22cd12af51d7bf167d26146e396e32048ae623 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 1 Apr 2021 22:12:02 +0000 Subject: [PATCH 12/54] Moved code between files. --- services/mirror/mirror.go | 35 --------------------------------- services/mirror/mirror_pull.go | 36 +++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 3c3a57616234..4af446dd45b1 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -10,10 +10,8 @@ import ( "strings" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" ) @@ -21,39 +19,6 @@ import ( // mirrorQueue holds an UniqueQueue object of the mirror var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) -// UpdateAddress writes new address to Git repository and database -func UpdateAddress(m *models.Mirror, addr string) error { - repoPath := m.Repo.RepoPath() - // Remove old remote - _, err := git.NewCommand("remote", "rm", pullMirrorRemoteName).RunInDir(repoPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - - _, err = git.NewCommand("remote", "add", pullMirrorRemoteName, "--mirror=fetch", addr).RunInDir(repoPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - - if m.Repo.HasWiki() { - wikiPath := m.Repo.WikiPath() - wikiRemotePath := repo_module.WikiRemoteURL(addr) - // Remove old remote of wiki - _, err := git.NewCommand("remote", "rm", pullMirrorRemoteName).RunInDir(wikiPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - - _, err = git.NewCommand("remote", "add", pullMirrorRemoteName, "--mirror=fetch", wikiRemotePath).RunInDir(wikiPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - } - - m.Repo.OriginalURL = addr - return models.UpdateRepositoryCols(m.Repo, "original_url") -} - // Update checks and updates mirror repositories. func Update(ctx context.Context) error { log.Trace("Doing: Update") diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index a7ee5c10e048..a1829e1d193b 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -26,6 +26,40 @@ const pullMirrorRemoteName = "origin" // gitShortEmptySha Git short empty SHA const gitShortEmptySha = "0000000" +// UpdateAddress writes new address to Git repository and database +func UpdateAddress(m *models.Mirror, addr string) error { + remoteName := m.GetRemoteName() + repoPath := m.Repo.RepoPath() + // Remove old remote + _, err := git.NewCommand("remote", "rm", remoteName).RunInDir(repoPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + + _, err = git.NewCommand("remote", "add", remoteName, "--mirror=fetch", addr).RunInDir(repoPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + + if m.Repo.HasWiki() { + wikiPath := m.Repo.WikiPath() + wikiRemotePath := repo_module.WikiRemoteURL(addr) + // Remove old remote of wiki + _, err := git.NewCommand("remote", "rm", remoteName).RunInDir(wikiPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + + _, err = git.NewCommand("remote", "add", remoteName, "--mirror=fetch", wikiRemotePath).RunInDir(wikiPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + } + + m.Repo.OriginalURL = addr + return models.UpdateRepositoryCols(m.Repo, "original_url") +} + // mirrorSyncResult contains information of a updated reference. // If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. // If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. From 56040872972d3401d30dae0c9b6c67fda583caee Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 1 Apr 2021 22:12:47 +0000 Subject: [PATCH 13/54] Lint --- models/migrations/v178.go | 6 +++--- modules/git/remote.go | 2 +- modules/templates/helper.go | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/migrations/v178.go b/models/migrations/v178.go index 7205750cdf30..04cd97775b3d 100644 --- a/models/migrations/v178.go +++ b/models/migrations/v178.go @@ -15,10 +15,10 @@ import ( func createPushMirrorTable(x *xorm.Engine) error { type PushMirror struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"INDEX"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` RemoteName string - + Interval time.Duration UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` diff --git a/modules/git/remote.go b/modules/git/remote.go index a36065ecfbae..7ba2b35a5ed3 100644 --- a/modules/git/remote.go +++ b/modules/git/remote.go @@ -23,7 +23,7 @@ func GetRemoteAddress(repoPath, remoteName string) (*url.URL, error) { if err != nil { return nil, err } - + if len(result) > 0 { result = result[:len(result)-1] } diff --git a/modules/templates/helper.go b/modules/templates/helper.go index fa1354f4f87b..9070657e28c6 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -294,7 +294,7 @@ func NewFuncMap() []template.FuncMap { } return float32(n) * 100 / float32(sum) }, - "CommentMustAsDiff": gitdiff.CommentMustAsDiff, + "CommentMustAsDiff": gitdiff.CommentMustAsDiff, "MirrorRemoteAddress": mirrorRemoteAddress, "CommitType": func(commit interface{}) string { switch commit.(type) { @@ -950,7 +950,7 @@ type remoteAddress struct { Password string } -func mirrorRemoteAddress(m models.RemoteMirrorer) (remoteAddress) { +func mirrorRemoteAddress(m models.RemoteMirrorer) remoteAddress { a := remoteAddress{} u, err := git.GetRemoteAddress(m.GetRepository().RepoPath(), m.GetRemoteName()) @@ -965,6 +965,6 @@ func mirrorRemoteAddress(m models.RemoteMirrorer) (remoteAddress) { } u.User = nil a.Address = u.String() - + return a -} \ No newline at end of file +} From 84a9bd846db34ad71de072f5a4f7e2422430c86d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 3 Apr 2021 22:08:35 +0000 Subject: [PATCH 14/54] Changed sanitizer interface. --- modules/task/migrate.go | 2 +- modules/util/sanitize.go | 63 +++++++------ modules/util/sanitize_test.go | 159 ++++++++++++++++++++++++++++++--- routers/api/v1/repo/migrate.go | 2 +- routers/repo/migrate.go | 2 +- services/mirror/mirror_pull.go | 53 ++++------- 6 files changed, 207 insertions(+), 74 deletions(-) diff --git a/modules/task/migrate.go b/modules/task/migrate.go index 57424abac38c..fe9b984d4407 100644 --- a/modules/task/migrate.go +++ b/modules/task/migrate.go @@ -118,7 +118,7 @@ func runMigrateTask(t *models.Task) (err error) { } // remoteAddr may contain credentials, so we sanitize it - err = util.URLSanitizedError(err, opts.CloneAddr) + err = util.NewStringURLSanitizedError(err, opts.CloneAddr, true) if strings.Contains(err.Error(), "Authentication failed") || strings.Contains(err.Error(), "could not read Username") { return fmt.Errorf("Authentication failed: %v", err.Error()) diff --git a/modules/util/sanitize.go b/modules/util/sanitize.go index a4f5479dfb74..de59ffaa2e5d 100644 --- a/modules/util/sanitize.go +++ b/modules/util/sanitize.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -9,40 +9,53 @@ import ( "strings" ) -// urlSafeError wraps an error whose message may contain a sensitive URL -type urlSafeError struct { - err error - unsanitizedURL string +const userPlaceholder = "sanitized-credential" +const unparsableURL = "(unparsable url)" + +type sanitizedError struct { + err error + replacer *strings.Replacer } -func (err urlSafeError) Error() string { - return SanitizeMessage(err.err.Error(), err.unsanitizedURL) +func (err sanitizedError) Error() string { + return err.replacer.Replace(err.err.Error()) } -// URLSanitizedError returns the sanitized version an error whose message may -// contain a sensitive URL -func URLSanitizedError(err error, unsanitizedURL string) error { - return urlSafeError{err: err, unsanitizedURL: unsanitizedURL} +// NewSanitizedError wraps an error and replaces all old, new string pairs in the message text. +func NewSanitizedError(err error, oldnew ...string) error { + return sanitizedError{err: err, replacer: strings.NewReplacer(oldnew...)} } -// SanitizeMessage sanitizes a message which may contains a sensitive URL -func SanitizeMessage(message, unsanitizedURL string) string { - sanitizedURL := SanitizeURLCredentials(unsanitizedURL, true) - return strings.ReplaceAll(message, unsanitizedURL, sanitizedURL) +// NewURLSanitizedError wraps an error and replaces the url credential or removes them. +func NewURLSanitizedError(err error, u *url.URL, usePlaceholder bool) error { + return sanitizedError{err: err, replacer: NewURLSanitizer(u, usePlaceholder)} } -// SanitizeURLCredentials sanitizes a url, either removing user credentials -// or replacing them with a placeholder. -func SanitizeURLCredentials(unsanitizedURL string, usePlaceholder bool) string { - u, err := url.Parse(unsanitizedURL) - if err != nil { - // don't log the error, since it might contain unsanitized URL. - return "(unparsable url)" - } +// NewStringURLSanitizedError wraps an error and replaces the url credential or removes them. +// If the url can't get parsed it gets replaced with a placeholder string. +func NewStringURLSanitizedError(err error, unsanitizedURL string, usePlaceholder bool) error { + return sanitizedError{err: err, replacer: NewStringURLSanitizer(unsanitizedURL, usePlaceholder)} +} + +// NewURLSanitizer creates a replacer for the url with the credential sanitized or removed. +func NewURLSanitizer(u *url.URL, usePlaceholder bool) *strings.Replacer { + old := u.String() + if u.User != nil && usePlaceholder { - u.User = url.User("") + u.User = url.User(userPlaceholder) } else { u.User = nil } - return u.String() + return strings.NewReplacer(old, u.String()) +} + +// NewStringURLSanitizer creates a replacer for the url with the credential sanitized or removed. +// If the url can't get parsed it gets replaced with a placeholder string +func NewStringURLSanitizer(unsanitizedURL string, usePlaceholder bool) *strings.Replacer { + u, err := url.Parse(unsanitizedURL) + if err != nil { + // don't log the error, since it might contain unsanitized URL. + return strings.NewReplacer(unsanitizedURL, unparsableURL) + } + return NewURLSanitizer(u, usePlaceholder) } diff --git a/modules/util/sanitize_test.go b/modules/util/sanitize_test.go index 4f07100675b0..578f75f5188f 100644 --- a/modules/util/sanitize_test.go +++ b/modules/util/sanitize_test.go @@ -1,25 +1,164 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package util import ( + "errors" "testing" "github.com/stretchr/testify/assert" ) -func TestSanitizeURLCredentials(t *testing.T) { - var kases = map[string]string{ - "https://github.com/go-gitea/test_repo.git": "https://github.com/go-gitea/test_repo.git", - "https://mytoken@github.com/go-gitea/test_repo.git": "https://github.com/go-gitea/test_repo.git", - "http://github.com/go-gitea/test_repo.git": "http://github.com/go-gitea/test_repo.git", - "/test/repos/repo1": "/test/repos/repo1", - "git@github.com:go-gitea/test_repo.git": "(unparsable url)", +func TestNewSanitizedError(t *testing.T) { + err := errors.New("error while secret on test") + err2 := NewSanitizedError(err) + assert.Equal(t, err.Error(), err2.Error()) + + var cases = []struct { + input error + oldnew []string + expected string + }{ + // case 0 + { + errors.New("error while secret on test"), + []string{"secret", "replaced"}, + "error while replaced on test", + }, + // case 1 + { + errors.New("error while sec-ret on test"), + []string{"secret", "replaced"}, + "error while sec-ret on test", + }, } - for source, value := range kases { - assert.EqualValues(t, value, SanitizeURLCredentials(source, false)) + for n, c := range cases { + err := NewSanitizedError(c.input, c.oldnew...) + + assert.Equal(t, c.expected, err.Error(), "case %d: error should match", n) + } +} + +func TestNewStringURLSanitizer(t *testing.T) { + var cases = []struct { + input string + placeholder bool + expected string + }{ + // case 0 + { + "https://github.com/go-gitea/test_repo.git", + true, + "https://github.com/go-gitea/test_repo.git", + }, + // case 1 + { + "https://github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 2 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 3 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 4 + { + "https://user:password@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 5 + { + "https://user:password@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 6 + { + "https://gi\nthub.com/go-gitea/test_repo.git", + false, + unparsableURL, + }, + } + + for n, c := range cases { + // uses NewURLSanitizer internally + result := NewStringURLSanitizer(c.input, c.placeholder).Replace(c.input) + + assert.Equal(t, c.expected, result, "case %d: error should match", n) + } +} + +func TestNewStringURLSanitizedError(t *testing.T) { + var cases = []struct { + input string + placeholder bool + expected string + }{ + // case 0 + { + "https://github.com/go-gitea/test_repo.git", + true, + "https://github.com/go-gitea/test_repo.git", + }, + // case 1 + { + "https://github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 2 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 3 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 4 + { + "https://user:password@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 5 + { + "https://user:password@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 6 + { + "https://gi\nthub.com/go-gitea/test_repo.git", + false, + unparsableURL, + }, + } + + encloseText := func(input string) string { + return "test " + input + " test" + } + + for n, c := range cases { + err := errors.New(encloseText(c.input)) + + result := NewStringURLSanitizedError(err, c.input, c.placeholder) + + assert.Equal(t, encloseText(c.expected), result.Error(), "case %d: error should match", n) } } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index 1eefa78d570a..fa8d797010db 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -233,7 +233,7 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA case base.IsErrNotSupported(err): ctx.Error(http.StatusUnprocessableEntity, "", err) default: - err = util.URLSanitizedError(err, remoteAddr) + err = util.NewStringURLSanitizedError(err, remoteAddr, true) if strings.Contains(err.Error(), "Authentication failed") || strings.Contains(err.Error(), "Bad credentials") || strings.Contains(err.Error(), "could not read Username") { diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 6b4e7852ae28..81c78549584e 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -99,7 +99,7 @@ func handleMigrateError(ctx *context.Context, owner *models.User, err error, nam ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form) default: remoteAddr, _ := auth.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword) - err = util.URLSanitizedError(err, remoteAddr) + err = util.NewStringURLSanitizedError(err, remoteAddr, true) if strings.Contains(err.Error(), "Authentication failed") || strings.Contains(err.Error(), "Bad credentials") || strings.Contains(err.Error(), "could not read Username") { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index a1829e1d193b..5e297fc24a24 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -21,8 +21,6 @@ import ( "code.gitea.io/gitea/modules/util" ) -const pullMirrorRemoteName = "origin" - // gitShortEmptySha Git short empty SHA const gitShortEmptySha = "0000000" @@ -160,21 +158,18 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { RunInDirTimeoutPipeline(timeout, repoPath, &stdoutBuilder, &stderrBuilder); err != nil { stdout := stdoutBuilder.String() stderr := stderrBuilder.String() + // sanitize the output, since it may contain the remote address, which may // contain a password - stderrMessage, sanitizeErr := sanitizeOutput(stderr, repoPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) - log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderr, err) - return nil, false - } - stdoutMessage, err := sanitizeOutput(stdout, repoPath) - if err != nil { - log.Error("sanitizeOutput failed: %v", sanitizeErr) - log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderrMessage, err) - return nil, false + remoteAddr, remoteErr := git.GetRemoteAddress(repoPath, m.GetRemoteName()) + if remoteErr != nil { + log.Error("GetRemoteAddress Error %v", remoteErr) } + sanitizer := util.NewURLSanitizer(remoteAddr, true) + stderrMessage := sanitizer.Replace(stderr) + stdoutMessage := sanitizer.Replace(stdout) + log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, stderrMessage) if err = models.CreateRepositoryNotice(desc); err != nil { @@ -211,21 +206,19 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { RunInDirTimeoutPipeline(timeout, wikiPath, &stdoutBuilder, &stderrBuilder); err != nil { stdout := stdoutBuilder.String() stderr := stderrBuilder.String() + // sanitize the output, since it may contain the remote address, which may // contain a password - stderrMessage, sanitizeErr := sanitizeOutput(stderr, repoPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) - log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderr, err) - return nil, false - } - stdoutMessage, err := sanitizeOutput(stdout, repoPath) - if err != nil { - log.Error("sanitizeOutput failed: %v", sanitizeErr) - log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdout, stderrMessage, err) - return nil, false + + remoteAddr, remoteErr := git.GetRemoteAddress(wikiPath, m.GetRemoteName()) + if remoteErr != nil { + log.Error("GetRemoteAddress Error %v", remoteErr) } + sanitizer := util.NewURLSanitizer(remoteAddr, true) + stderrMessage := sanitizer.Replace(stderr) + stdoutMessage := sanitizer.Replace(stdout) + log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) desc := fmt.Sprintf("Failed to update mirror repository wiki '%s': %s", wikiPath, stderrMessage) if err = models.CreateRepositoryNotice(desc); err != nil { @@ -445,15 +438,3 @@ func checkAndUpdateEmptyRepository(m *models.Mirror, gitRepo *git.Repository, re } return true } - -// sanitizeOutput sanitizes output of a command, replacing occurrences of the -// repository's remote address with a sanitized version. -func sanitizeOutput(output, repoPath string) (string, error) { - remoteAddr, err := readRemoteAddress(repoPath, pullMirrorRemoteName) - if err != nil { - // if we're unable to load the remote address, then we're unable to - // sanitize. - return "", err - } - return util.SanitizeMessage(output, remoteAddr), nil -} From ca14354b500422df5fe8a69ad43cf79cdc1d0fb2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 3 Apr 2021 22:53:57 +0000 Subject: [PATCH 15/54] Added push mirror backend methods. --- models/repo_mirror.go | 3 +- services/mirror/mirror_push.go | 60 +++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 249b70679461..7075789f6ff0 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -14,7 +14,7 @@ import ( "xorm.io/xorm" ) -// RemoteMirrorer +// RemoteMirrorer defines base methods for pull/push mirrors. type RemoteMirrorer interface { GetRepository() *Repository GetRemoteName() string @@ -65,7 +65,6 @@ func (m *Mirror) GetRemoteName() string { return "origin" } - // ScheduleNextUpdate calculates and sets next update time. func (m *Mirror) ScheduleNextUpdate() { if m.Interval != 0 { diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 8cb970467e3e..c4a4ff2ca4d8 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -5,7 +5,7 @@ package mirror import ( - "fmt" + "errors" "strconv" "time" @@ -14,6 +14,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" ) // AddPushMirrorRemote registers the push mirror remote. @@ -22,7 +24,7 @@ func AddPushMirrorRemote(m *models.PushMirror, addr string) error { return err } - if repo.HasWiki() { + if m.Repo.HasWiki() { wikiRemotePath := repository.WikiRemoteURL(addr) if len(wikiRemotePath) > 0 { if _, err := git.NewCommand("remote", "add", m.RemoteName, wikiRemotePath).RunInDir(m.Repo.WikiPath()); err != nil { @@ -30,28 +32,30 @@ func AddPushMirrorRemote(m *models.PushMirror, addr string) error { } } } - + return nil } // RemovePushMirrorRemote removes the push mirror remote. func RemovePushMirrorRemote(m *models.PushMirror) error { cmd := git.NewCommand("remote", "rm", m.RemoteName) - + if _, err := cmd.RunInDir(m.Repo.RepoPath()); err != nil { return err } - if _, err := cmd.RunInDir(m.Repo.WikiPath()); err != nil { - // The wiki remote may not exist - log.Warning("Wiki Remote[%d] could not be removed: %v", m.ID, err) + if m.Repo.HasWiki() { + if _, err := cmd.RunInDir(m.Repo.WikiPath()); err != nil { + // The wiki remote may not exist + log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err) + } } - + return nil } func syncPushMirror(mirrorID string) { - log.Trace("SyncPushMirror [mirror_id: %v]", mirrorID) + log.Trace("SyncPushMirror [mirror: %s]", mirrorID) defer func() { err := recover() if err == nil { @@ -64,50 +68,52 @@ func syncPushMirror(mirrorID string) { id, _ := strconv.ParseInt(mirrorID, 10, 64) m, err := models.GetPushMirrorByID(id) if err != nil { - log.Error("GetPushMirrorByID [%d]: %v", id, err) + log.Error("GetPushMirrorByID [%s]: %v", mirrorID, err) return } - log.Trace("SyncPushMirror [repo: %-v]: Running Sync", m.Repo) + m.UpdatedUnix = timeutil.TimeStampNow() + + log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Running Sync", mirrorID, m.Repo) err = runPushSync(m) if err != nil { - log.Error("SyncPushMirror [%d]: %v", id, err) - return + m.LastError = err.Error() } - log.Trace("SyncPushMirror [repo: %-v]: Scheduling next update", m.Repo) + log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Scheduling next update", mirrorID, m.Repo) m.ScheduleNextUpdate() if err = models.UpdatePushMirror(m); err != nil { - log.Error("UpdatePushMirror [%d]: %v", id, err) - return + log.Error("UpdatePushMirror [%s]: %v", mirrorID, err) } - log.Trace("SyncPushMirror [repo: %-v]: Successfully updated", m.Repo) + log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Finished", mirrorID, m.Repo) } -// runPushSync returns true if sync finished without error. func runPushSync(m *models.PushMirror) error { - repoPath := m.Repo.RepoPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second - log.Trace("SyncPushMirror [repo: %-v]: running git push...", m.Repo) - performPush := func(path string) error { + log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + if err := git.Push(path, git.PushOptions{ Remote: m.RemoteName, Force: true, Mirror: true, Timeout: timeout, }); err != nil { - if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { - return err + log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) + + remoteAddr, remoteErr := git.GetRemoteAddress(path, m.RemoteName) + if remoteErr != nil { + log.Error("GetRemoteAddress(%s) Error %v", path, remoteErr) + return errors.New("Unexpected error") } - return fmt.Errorf("Error pushing remote %s to %s: %v", m.RemoteName, path, err) + return util.NewURLSanitizedError(err, remoteAddr, true) } return nil } - err := performPush(repoPath) + err := performPush(m.Repo.RepoPath()) if err != nil { return err } @@ -115,9 +121,9 @@ func runPushSync(m *models.PushMirror) error { // TODO LFS if m.Repo.HasWiki() { - err := performPush(m.Repo.WikiPath()); + err := performPush(m.Repo.WikiPath()) if err != nil { - return nil + return err } } From a9b19612bac3133c579cdb3792bebaf9b8b1f575 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 3 Apr 2021 22:58:57 +0000 Subject: [PATCH 16/54] Fix --- models/repo.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/repo.go b/models/repo.go index 329e63cc421b..80b80fbe8711 100644 --- a/models/repo.go +++ b/models/repo.go @@ -254,7 +254,12 @@ func (repo *Repository) SanitizedOriginalURL() string { if repo.OriginalURL == "" { return "" } - return util.SanitizeURLCredentials(repo.OriginalURL, false) + u, err := url.Parse(repo.OriginalURL) + if err != nil { + return "" + } + u.User = nil + return u.String() } // ColorFormat returns a colored string to represent this repo From ca2c530eb56f47c764eb22211cb1afb4e46cb957 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 4 Apr 2021 13:43:38 +0000 Subject: [PATCH 17/54] Only update the mirror remote. --- services/mirror/mirror_pull.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 5e297fc24a24..dc442125471a 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -150,6 +150,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { if m.EnablePrune { gitArgs = append(gitArgs, "--prune") } + gitArgs = append(gitArgs, m.GetRemoteName()) stdoutBuilder := strings.Builder{} stderrBuilder := strings.Builder{} @@ -201,7 +202,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) stderrBuilder.Reset() stdoutBuilder.Reset() - if err := git.NewCommand("remote", "update", "--prune"). + if err := git.NewCommand("remote", "update", "--prune", m.GetRemoteName()). SetDescription(fmt.Sprintf("Mirror.runSync Wiki: %s ", m.Repo.FullName())). RunInDirTimeoutPipeline(timeout, wikiPath, &stdoutBuilder, &stderrBuilder); err != nil { stdout := stdoutBuilder.String() From 84d13523bb5ff37e760e06925d26f5c6236574bd Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 5 Apr 2021 19:30:12 +0000 Subject: [PATCH 18/54] Limit refs on push. --- services/mirror/mirror_push.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index c4a4ff2ca4d8..e5a1ed80fab2 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -20,14 +20,27 @@ import ( // AddPushMirrorRemote registers the push mirror remote. func AddPushMirrorRemote(m *models.PushMirror, addr string) error { - if _, err := git.NewCommand("remote", "add", m.RemoteName, addr).RunInDir(m.Repo.RepoPath()); err != nil { + addRemoteAndConfig := func(addr, path string) error { + if _, err := git.NewCommand("remote", "add", "--mirror=push", m.RemoteName, addr).RunInDir(path); err != nil { + return err + } + if _, err := git.NewCommand("config", "--add", "remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunInDir(path); err != nil { + return err + } + if _, err := git.NewCommand("config", "--add", "remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunInDir(path); err != nil { + return err + } + return nil + } + + if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil { return err } if m.Repo.HasWiki() { - wikiRemotePath := repository.WikiRemoteURL(addr) - if len(wikiRemotePath) > 0 { - if _, err := git.NewCommand("remote", "add", m.RemoteName, wikiRemotePath).RunInDir(m.Repo.WikiPath()); err != nil { + wikiRemoteURL := repository.WikiRemoteURL(addr) + if len(wikiRemoteURL) > 0 { + if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil { return err } } From 806408a6696c7f1f1d3357bc727a043ef795e10d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 5 Apr 2021 21:12:15 +0000 Subject: [PATCH 19/54] Added UI part. --- models/repo.go | 19 ++- modules/context/repo.go | 6 +- modules/forms/repo_form.go | 25 ++-- options/locale/locale_en-US.ini | 13 +- routers/repo/setting.go | 161 ++++++++++++++++++---- services/mirror/mirror.go | 5 + templates/repo/settings/options.tmpl | 194 +++++++++++++++++++-------- 7 files changed, 325 insertions(+), 98 deletions(-) diff --git a/models/repo.go b/models/repo.go index 80b80fbe8711..a5e579b167f9 100644 --- a/models/repo.go +++ b/models/repo.go @@ -215,12 +215,13 @@ type Repository struct { NumClosedProjects int `xorm:"NOT NULL DEFAULT 0"` NumOpenProjects int `xorm:"-"` - IsPrivate bool `xorm:"INDEX"` - IsEmpty bool `xorm:"INDEX"` - IsArchived bool `xorm:"INDEX"` - IsMirror bool `xorm:"INDEX"` - *Mirror `xorm:"-"` - Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` + IsPrivate bool `xorm:"INDEX"` + IsEmpty bool `xorm:"INDEX"` + IsArchived bool `xorm:"INDEX"` + IsMirror bool `xorm:"INDEX"` + *Mirror `xorm:"-"` + PushMirrors []*PushMirror `xorm:"-"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` RenderingMetas map[string]string `xorm:"-"` DocumentRenderingMetas map[string]string `xorm:"-"` @@ -661,6 +662,12 @@ func (repo *Repository) GetMirror() (err error) { return err } +// LoadPushMirrors populates the repository push mirrors. +func (repo *Repository) LoadPushMirrors() (err error) { + repo.PushMirrors, err = GetPushMirrorsByRepoID(repo.ID) + return err +} + // GetBaseRepo populates repo.BaseRepo for a fork repository and // returns an error on failure (NOTE: no error is returned for // non-fork repositories, and BaseRepo will be left untouched) diff --git a/modules/context/repo.go b/modules/context/repo.go index ba3cfe9bf256..6386fdc9c303 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -360,13 +360,17 @@ func repoAssignment(ctx *Context, repo *models.Repository) { var err error ctx.Repo.Mirror, err = models.GetMirrorByRepoID(repo.ID) if err != nil { - ctx.ServerError("GetMirror", err) + ctx.ServerError("GetMirrorByRepoID", err) return } ctx.Data["MirrorEnablePrune"] = ctx.Repo.Mirror.EnablePrune ctx.Data["MirrorInterval"] = ctx.Repo.Mirror.Interval ctx.Data["Mirror"] = ctx.Repo.Mirror } + if err = repo.LoadPushMirrors(); err != nil { + ctx.ServerError("LoadPushMirrors", err) + return + } ctx.Repo.Repository = repo ctx.Data["RepoName"] = ctx.Repo.Repository.Name diff --git a/modules/forms/repo_form.go b/modules/forms/repo_form.go index 6c7c9bea138d..a745e684c994 100644 --- a/modules/forms/repo_form.go +++ b/modules/forms/repo_form.go @@ -111,16 +111,21 @@ func ParseRemoteAddr(remoteAddr, authUsername, authPassword string) (string, err // RepoSettingForm form for changing repository settings type RepoSettingForm struct { - RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"` - Description string `binding:"MaxSize(255)"` - Website string `binding:"ValidUrl;MaxSize(255)"` - Interval string - MirrorAddress string - MirrorUsername string - MirrorPassword string - Private bool - Template bool - EnablePrune bool + RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"` + Description string `binding:"MaxSize(255)"` + Website string `binding:"ValidUrl;MaxSize(255)"` + Interval string + MirrorAddress string + MirrorUsername string + MirrorPassword string + PushMirrorID string + PushMirrorAddress string + PushMirrorUsername string + PushMirrorPassword string + PushMirrorInterval string + Private bool + Template bool + EnablePrune bool // Advanced settings EnableWiki bool diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e679e1e87497..5f90e8e0ca6f 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -91,6 +91,7 @@ loading = Loading… step1 = Step 1: step2 = Step 2: +error = Error error404 = The page you are trying to reach either does not exist or you are not authorized to view it. [error] @@ -715,7 +716,7 @@ mirror_prune_desc = Remove obsolete remote-tracking references mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable automatic sync. mirror_interval_invalid = The mirror interval is not valid. mirror_address = Clone From URL -mirror_address_desc = Put any required credentials in the Clone Authorization section. +mirror_address_desc = Put any required credentials in the Authorization section. mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly. mirror_address_protocol_invalid = The provided url is invalid. Only http(s):// or git:// locations can be mirrored from. mirror_last_synced = Last Synchronized @@ -771,7 +772,7 @@ form.reach_limit_of_creation_n = You have already reached your limit of %d repos form.name_reserved = The repository name '%s' is reserved. form.name_pattern_not_allowed = The pattern '%s' is not allowed in a repository name. -need_auth = Clone Authorization +need_auth = Authorization migrate_options = Migration Options migrate_service = Migration Service migrate_options_mirror_helper = This repository will be a mirror @@ -1520,6 +1521,14 @@ settings.hooks = Webhooks settings.githooks = Git Hooks settings.basic_settings = Basic Settings settings.mirror_settings = Mirror Settings +settings.mirror_settings.mirrored_repository = Mirrored repository +settings.mirror_settings.direction = Direction +settings.mirror_settings.direction.pull = Pull +settings.mirror_settings.direction.push = Push +settings.mirror_settings.last_update = Last update +settings.mirror_settings.push_mirror.none = No push mirrors configured +settings.mirror_settings.push_mirror.remote_url = Git Remote Repository URL +settings.mirror_settings.push_mirror.add = Add Push Mirror settings.sync_mirror = Synchronize Now settings.mirror_sync_in_progress = Mirror synchronization is in progress. Check back in a minute. settings.email_notifications.enable = Enable Email Notifications diff --git a/routers/repo/setting.go b/routers/repo/setting.go index dc14b69b3b0b..fd177f8c6f41 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io/ioutil" + "strconv" "strings" "time" @@ -16,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" auth "code.gitea.io/gitea/modules/forms" + "code.gitea.io/gitea/modules/generate" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" @@ -46,6 +48,8 @@ func Settings(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.settings") ctx.Data["PageIsSettingsOptions"] = true ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate + ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors + ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath()) ctx.Data["SigningKeyAvailable"] = len(signing) > 0 @@ -169,30 +173,8 @@ func SettingsPost(ctx *context.Context) { err = migrations.IsMigrateURLAllowed(address, ctx.User) } if err != nil { - if models.IsErrInvalidCloneAddr(err) { - ctx.Data["Err_MirrorAddress"] = true - addrErr := err.(*models.ErrInvalidCloneAddr) - switch { - case addrErr.IsProtocolInvalid: - ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, &form) - case addrErr.IsURLError: - ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, &form) - case addrErr.IsPermissionDenied: - if addrErr.LocalPath { - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, &form) - } else if len(addrErr.PrivateNet) == 0 { - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, &form) - } else { - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, &form) - } - case addrErr.IsInvalidPath: - ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, &form) - default: - ctx.ServerError("Unknown error", err) - } - } ctx.Data["Err_MirrorAddress"] = true - ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, &form) + handleMirrorAddressError(ctx, err, form) return } @@ -215,6 +197,95 @@ func SettingsPost(ctx *context.Context) { ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress")) ctx.Redirect(repo.Link() + "/settings") + case "push-mirror-sync": + m, err := selectPushMirrorByForm(form, repo) + if err != nil { + ctx.NotFound("", nil) + return + } + + mirror_service.AddPushMirrorToQueue(m.ID) + + ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress")) + ctx.Redirect(repo.Link() + "/settings") + + case "push-mirror-remove": + // This section doesn't require repo_name/RepoName to be set in the form, don't show it + // as an error on the UI for this action + ctx.Data["Err_RepoName"] = nil + + m, err := selectPushMirrorByForm(form, repo) + if err != nil { + ctx.NotFound("", nil) + return + } + + if err = mirror_service.RemovePushMirrorRemote(m); err != nil { + ctx.ServerError("RemovePushMirrorRemote", err) + return + } + + if err = models.DeletePushMirrorByID(m.ID); err != nil { + ctx.ServerError("DeletePushMirrorByID", err) + return + } + + ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success")) + ctx.Redirect(repo.Link() + "/settings") + + case "push-mirror-add": + // This section doesn't require repo_name/RepoName to be set in the form, don't show it + // as an error on the UI for this action + ctx.Data["Err_RepoName"] = nil + + interval, err := time.ParseDuration(form.PushMirrorInterval) + if err != nil || (interval != 0 && interval < setting.Mirror.MinInterval) { + ctx.Data["Err_PushMirrorInterval"] = true + ctx.RenderWithErr(ctx.Tr("repo.mirror_interval_invalid"), tplSettingsOptions, &form) + return + } + + address, err := auth.ParseRemoteAddr(form.PushMirrorAddress, form.PushMirrorUsername, form.PushMirrorPassword) + if err == nil { + err = migrations.IsMigrateURLAllowed(address, ctx.User) + } + if err != nil { + ctx.Data["Err_PushMirrorAddress"] = true + handleMirrorAddressError(ctx, err, form) + return + } + + remoteSuffix, err := generate.GetRandomString(10) + if err != nil { + ctx.ServerError("GetRandomString", err) + return + } + + m := &models.PushMirror{ + RepoID: repo.ID, + Repo: repo, + RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), + Interval: interval, + } + if interval != 0 { + m.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(interval) + } + if err := models.InsertPushMirror(m); err != nil { + ctx.ServerError("InsertPushMirror", err) + return + } + + if err := mirror_service.AddPushMirrorRemote(m, address); err != nil { + if err := models.DeletePushMirrorByID(m.ID); err != nil { + log.Error("DeletePushMirrorByID %v", err) + } + ctx.ServerError("AddPushMirrorRemote", err) + return + } + + ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success")) + ctx.Redirect(repo.Link() + "/settings") + case "advanced": var repoChanged bool var units []models.RepoUnit @@ -613,6 +684,31 @@ func SettingsPost(ctx *context.Context) { } } +func handleMirrorAddressError(ctx *context.Context, err error, form *auth.RepoSettingForm) { + if models.IsErrInvalidCloneAddr(err) { + addrErr := err.(*models.ErrInvalidCloneAddr) + switch { + case addrErr.IsProtocolInvalid: + ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, form) + case addrErr.IsURLError: + ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, form) + case addrErr.IsPermissionDenied: + if addrErr.LocalPath { + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, form) + } else if len(addrErr.PrivateNet) == 0 { + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form) + } else { + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, form) + } + case addrErr.IsInvalidPath: + ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, form) + default: + ctx.ServerError("Unknown error", err) + } + } + ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, form) +} + // Collaboration render a repository's collaboration page func Collaboration(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.settings") @@ -1008,3 +1104,22 @@ func SettingsDeleteAvatar(ctx *context.Context) { } ctx.Redirect(ctx.Repo.RepoLink + "/settings") } + +func selectPushMirrorByForm(form *auth.RepoSettingForm, repo *models.Repository) (*models.PushMirror, error) { + id, err := strconv.ParseInt(form.PushMirrorID, 10, 64) + if err != nil { + return nil, err + } + + if err = repo.LoadPushMirrors(); err != nil { + return nil, err + } + + for _, m := range repo.PushMirrors { + if m.ID == id { + return m, nil + } + } + + return nil, fmt.Errorf("PushMirror[%v] not associated to repository %v", id, repo) +} diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 4af446dd45b1..a93c30856d43 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -94,3 +94,8 @@ func InitSyncMirrors() { func StartToMirror(repoID int64) { go mirrorQueue.Add(fmt.Sprintf("pull %d", repoID)) } + +// AddPushMirrorToQueue adds the push mirror to the queue +func AddPushMirrorToQueue(mirrorID int64) { + go mirrorQueue.Add(fmt.Sprintf("push %d", mirrorID)) +} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 0d668684ef02..0aa4db503df4 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -70,66 +70,148 @@ - {{if .Repository.IsMirror}} + {{if or .Repository.IsMirror (not .DisabledMirrors)}}

{{.i18n.Tr "repo.settings.mirror_settings"}}

-
- {{.CsrfTokenHtml}} - -
- -
- - -
-
-
- - -
-
- - -

{{.i18n.Tr "repo.mirror_address_desc"}}

-
-
- -
-
- - -
- -
- - -
-
-
- -
- -
-
- -
- -
- {{.CsrfTokenHtml}} - -
- - {{.Mirror.UpdatedUnix.AsTime}} -
-
- -
-
+ + + + + + + + + + {{if .Repository.IsMirror}} + + + + + + + + + + + + + {{end}} + + {{range .Repository.PushMirrors}} + + {{$address := MirrorRemoteAddress .}} + + + + + + {{else}} + + + + {{end}} + + + + +
{{$.i18n.Tr "repo.settings.mirror_settings.mirrored_repository"}}{{$.i18n.Tr "repo.settings.mirror_settings.direction"}}{{$.i18n.Tr "repo.settings.mirror_settings.last_update"}}
{{(MirrorRemoteAddress .Mirror).Address}}{{$.i18n.Tr "repo.settings.mirror_settings.direction.pull"}}{{.Mirror.UpdatedUnix.AsTime}} +
+ {{.CsrfTokenHtml}} + + +
+
+
+ {{.CsrfTokenHtml}} + +
+ +
+ + +
+
+
+ + +
+ {{$address := MirrorRemoteAddress .Mirror}} +
+ + +

{{.i18n.Tr "repo.mirror_address_desc"}}

+
+
+ +
+
+ + +
+ +
+ + +
+
+
+
+ +
+
+
{{$address.Address}}{{$.i18n.Tr "repo.settings.mirror_settings.direction.push"}}{{.UpdatedUnix.AsTime}} {{if .LastError}}
{{$.i18n.Tr "error"}}
{{end}}		 +
+ {{$.CsrfTokenHtml}} + + + +
+
+ {{$.CsrfTokenHtml}} + + + +
+
{{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.none"}}
+
+ {{.CsrfTokenHtml}} + +
+ + +

{{.i18n.Tr "repo.mirror_address_desc"}}

+
+
+ +
+
+ + +
+ +
+ + +
+
+
+
+ + +
+
+ +
+
+
{{end}} From 79201d757536b8c344f9ca219b725349011c17cb Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 5 Apr 2021 23:40:00 +0200 Subject: [PATCH 20/54] Added missing table. --- models/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/models.go b/models/models.go index 73e65d828bdf..c48d9b0b025e 100644 --- a/models/models.go +++ b/models/models.go @@ -134,6 +134,7 @@ func init() { new(ProjectIssue), new(Session), new(RepoTransfer), + new(PushMirror), ) gonicNames := []string{"SSL", "UID"} From b78b83e23f6662768820fdf1e5570347d5f39c01 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 6 Apr 2021 16:18:00 +0200 Subject: [PATCH 21/54] Delete mirror if repository gets removed. --- models/repo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/repo.go b/models/repo.go index f766893c4313..e116d3fbb3a6 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1466,6 +1466,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { &Watch{RepoID: repoID}, &Star{RepoID: repoID}, &Mirror{RepoID: repoID}, + &PushMirror{RepoID: repoID}, &Milestone{RepoID: repoID}, &Release{RepoID: repoID}, &Collaboration{RepoID: repoID}, From c0d3c787b66bf82fee754c8647cb69f38a601072 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 17 Apr 2021 10:59:45 +0000 Subject: [PATCH 22/54] Changed signature. Handle object errors. --- modules/lfs/client.go | 2 +- modules/lfs/filesystem_client.go | 4 ++-- modules/lfs/http_client.go | 17 ++++++++++++++--- modules/lfs/shared.go | 4 ++-- modules/lfs/transferadapter.go | 14 ++++---------- modules/repository/repo.go | 2 +- 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index ae35919d770b..872535479793 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -12,7 +12,7 @@ import ( // Client is used to communicate with a LFS source type Client interface { - Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) + Download(ctx context.Context, p Pointer) (io.ReadCloser, error) } // NewClient creates a LFS client diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 3a51564a821b..c14290d2c833 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -34,8 +34,8 @@ func (c *FilesystemClient) objectPath(oid string) string { } // Download reads the specific LFS object from the target repository -func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { - objectPath := c.objectPath(oid) +func (c *FilesystemClient) Download(ctx context.Context, p Pointer) (io.ReadCloser, error) { + objectPath := c.objectPath(p.Oid) if _, err := os.Stat(objectPath); os.IsNotExist(err) { return nil, err diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index fb45defda1cf..dbe2c8e8f892 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -103,9 +103,9 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin } // Download reads the specific LFS object from the LFS server -func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { +func (c *HTTPClient) Download(ctx context.Context, p Pointer) (io.ReadCloser, error) { var objects []Pointer - objects = append(objects, Pointer{oid, size}) + objects = append(objects, p) result, err := c.batch(ctx, "download", objects) if err != nil { @@ -121,7 +121,18 @@ func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.R return nil, errors.New("lfs.HTTPClient.Download: No objects in result") } - content, err := transferAdapter.Download(ctx, result.Objects[0]) + object := result.Objects[0] + + if object.Error != nil { + return nil, errors.New(object.Error.Message) + } + + link, ok := object.Actions["download"] + if !ok { + return nil, errors.New("lfs.HTTPClient.Download: Action 'download' not found") + } + + content, err := transferAdapter.Download(ctx, link) if err != nil { return nil, err } diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 70b76d7512d5..ab850cee5c6c 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -49,14 +49,14 @@ type ObjectResponse struct { Error *ObjectError `json:"error,omitempty"` } -// Link provides a structure used to build a hypermedia representation of an HTTP link. +// Link provides a structure with informations about how to access a object. type Link struct { Href string `json:"href"` Header map[string]string `json:"header,omitempty"` ExpiresAt time.Time `json:"expires_at,omitempty"` } -// ObjectError defines the JSON structure returned to the client in case of an error +// ObjectError defines the JSON structure returned to the client in case of an error. type ObjectError struct { Code int `json:"code"` Message string `json:"message"` diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index ea3aff0000b9..01c80dc11c1d 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -15,8 +15,7 @@ import ( // TransferAdapter represents an adapter for downloading/uploading LFS objects type TransferAdapter interface { Name() string - Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) - //Upload(ctx context.Context, reader io.Reader) error + Download(ctx context.Context, l *Link) (io.ReadCloser, error) } // BasicTransferAdapter implements the "basic" adapter @@ -30,17 +29,12 @@ func (a *BasicTransferAdapter) Name() string { } // Download reads the download location and downloads the data -func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { - download, ok := r.Actions["download"] - if !ok { - return nil, errors.New("lfs.BasicTransferAdapter.Download: Action 'download' not found") - } - - req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil) +func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, "GET", l.Href, nil) if err != nil { return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err) } - for key, value := range download.Header { + for key, value := range l.Header { req.Header.Set(key, value) } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 50eb185daa9e..d9dc82b03917 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -346,7 +346,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi return nil } - stream, err := client.Download(ctx, pointerBlob.Oid, pointerBlob.Size) + stream, err := client.Download(ctx, pointerBlob.Pointer) if err != nil { return fmt.Errorf("StoreMissingLfsObjectsInRepository: LFS OID[%s] failed to download: %w", pointerBlob.Oid, err) } From 20de69cd4fa02e4b1632b46794267aa280ec671f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 17 Apr 2021 12:40:21 +0000 Subject: [PATCH 23/54] Added upload method. --- modules/lfs/client.go | 1 + modules/lfs/filesystem_client.go | 21 +++++-- modules/lfs/http_client.go | 92 +++++++++++++++++++++++------ modules/lfs/http_client_test.go | 19 ++++-- modules/lfs/shared.go | 7 +++ modules/lfs/transferadapter.go | 66 +++++++++++++++++++-- modules/lfs/transferadapter_test.go | 24 ++------ 7 files changed, 176 insertions(+), 54 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index 872535479793..fb1a843f0a08 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -13,6 +13,7 @@ import ( // Client is used to communicate with a LFS source type Client interface { Download(ctx context.Context, p Pointer) (io.ReadCloser, error) + Upload(ctx context.Context, p Pointer, content io.Reader) error } // NewClient creates a LFS client diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index c14290d2c833..9ba8b206aafe 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -33,18 +33,27 @@ func (c *FilesystemClient) objectPath(oid string) string { return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid) } -// Download reads the specific LFS object from the target repository +// Download reads the specific LFS object from the target path func (c *FilesystemClient) Download(ctx context.Context, p Pointer) (io.ReadCloser, error) { objectPath := c.objectPath(p.Oid) - if _, err := os.Stat(objectPath); os.IsNotExist(err) { - return nil, err + return os.Open(objectPath) +} + +// Upload writes the specific LFS object to the target path +func (c *FilesystemClient) Upload(ctx context.Context, p Pointer, r io.Reader) error { + objectPath := c.objectPath(p.Oid) + + if err := os.MkdirAll(filepath.Dir(objectPath), 0600); err != nil { + return err } - file, err := os.Open(objectPath) + f, err := os.Create(objectPath) if err != nil { - return nil, err + return err } - return file, nil + _, err = io.Copy(f, r) + + return err } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index dbe2c8e8f892..abfacdce9c7f 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -7,7 +7,6 @@ package lfs import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -16,6 +15,8 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + + jsoniter "github.com/json-iterator/go" ) // HTTPClient is used to communicate with the LFS server @@ -60,7 +61,7 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin request := &BatchRequest{operation, c.transferNames(), nil, objects} payload := new(bytes.Buffer) - err := json.NewEncoder(payload).Encode(request) + err := jsoniter.NewEncoder(payload).Encode(request) if err != nil { return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err) } @@ -90,7 +91,7 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin } var response BatchResponse - err = json.NewDecoder(res.Body).Decode(&response) + err = jsoniter.NewDecoder(res.Body).Decode(&response) if err != nil { return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err) } @@ -104,37 +105,92 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin // Download reads the specific LFS object from the LFS server func (c *HTTPClient) Download(ctx context.Context, p Pointer) (io.ReadCloser, error) { - var objects []Pointer - objects = append(objects, p) - - result, err := c.batch(ctx, "download", objects) + bc := batchContext{ + IsUpload: false, + Pointer: p, + } + err := c.performOperation(ctx, &bc) if err != nil { return nil, err } + return bc.DownloadResult, nil +} + +// Upload sends the specific LFS object to the LFS server +func (c *HTTPClient) Upload(ctx context.Context, p Pointer, r io.Reader) error { + bc := batchContext{ + IsUpload: true, + Pointer: p, + UploadContent: r, + } + return c.performOperation(ctx, &bc) +} + +type batchContext struct { + Pointer + IsUpload bool + + DownloadResult io.ReadCloser + UploadContent io.Reader +} + +func (c *HTTPClient) performOperation(ctx context.Context, bc *batchContext) error { + operation := "download" + if bc.IsUpload { + operation = "upload" + } + + result, err := c.batch(ctx, operation, []Pointer{bc.Pointer}) + if err != nil { + return err + } transferAdapter, ok := c.transfers[result.Transfer] if !ok { - return nil, fmt.Errorf("lfs.HTTPClient.Download Transferadapter not found: %s", result.Transfer) + return fmt.Errorf("LFS TransferAdapter not found: %s", result.Transfer) } if len(result.Objects) == 0 { - return nil, errors.New("lfs.HTTPClient.Download: No objects in result") + return errors.New("No LFS objects in result") } object := result.Objects[0] if object.Error != nil { - return nil, errors.New(object.Error.Message) + return errors.New(object.Error.Message) } - link, ok := object.Actions["download"] - if !ok { - return nil, errors.New("lfs.HTTPClient.Download: Action 'download' not found") - } + if bc.IsUpload { + if len(object.Actions) == 0 { + return nil + } - content, err := transferAdapter.Download(ctx, link) - if err != nil { - return nil, err + link, ok := object.Actions["upload"] + if !ok { + return errors.New("Action 'upload' not found") + } + + if err := transferAdapter.Upload(ctx, link, bc.UploadContent); err != nil { + return err + } + + link, ok = object.Actions["verify"] + if ok { + if err := transferAdapter.Verify(ctx, link, bc.Pointer); err != nil { + return err + } + } + } else { + link, ok := object.Actions["download"] + if !ok { + return errors.New("Action 'download' not found") + } + + var err error + bc.DownloadResult, err = transferAdapter.Download(ctx, link) + if err != nil { + return err + } } - return content, nil + return nil } diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 043aa0214e86..b2b684700f55 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -30,13 +30,20 @@ func (a *DummyTransferAdapter) Name() string { return "dummy" } -func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { +func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil } +func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, r io.Reader) error { + return nil +} + +func (a *DummyTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error { + return nil +} + func TestHTTPClientDownload(t *testing.T) { - oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041" - size := int64(6) + p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} roundTripHandler := func(req *http.Request) *http.Response { url := req.URL.String() @@ -57,8 +64,8 @@ func TestHTTPClientDownload(t *testing.T) { assert.Equal(t, "download", batchRequest.Operation) assert.Equal(t, 1, len(batchRequest.Objects)) - assert.Equal(t, oid, batchRequest.Objects[0].Oid) - assert.Equal(t, size, batchRequest.Objects[0].Size) + assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid) + assert.Equal(t, p.Size, batchRequest.Objects[0].Size) batchResponse := &BatchResponse{ Transfer: "dummy", @@ -134,7 +141,7 @@ func TestHTTPClientDownload(t *testing.T) { } client.transfers["dummy"] = dummy - _, err := client.Download(context.Background(), oid, size) + _, err := client.Download(context.Background(), p) if len(c.expectederror) > 0 { assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index ab850cee5c6c..8a5ea2abe34e 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -67,3 +67,10 @@ type PointerBlob struct { Hash string Pointer } + +// ErrorResponse describes the error to the client. +type ErrorResponse struct { + Message string + DocumentationUrl string `json:"documentation_url"` + RequestID string `json:"request_id"` +} diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 01c80dc11c1d..6a35d7b9e0af 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -5,17 +5,22 @@ package lfs import ( + "bytes" "context" "errors" "fmt" "io" "net/http" + + jsoniter "github.com/json-iterator/go" ) // TransferAdapter represents an adapter for downloading/uploading LFS objects type TransferAdapter interface { Name() string Download(ctx context.Context, l *Link) (io.ReadCloser, error) + Upload(ctx context.Context, l *Link, r io.Reader) error + Verify(ctx context.Context, l *Link, p Pointer) error } // BasicTransferAdapter implements the "basic" adapter @@ -30,9 +35,40 @@ func (a *BasicTransferAdapter) Name() string { // Download reads the download location and downloads the data func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { - req, err := http.NewRequestWithContext(ctx, "GET", l.Href, nil) + resp, err := a.performRequest(ctx, "GET", l, nil) if err != nil { - return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err) + return nil, err + } + return resp.Body, nil +} + +// Upload sends the content to the LFS server +func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, r io.Reader) error { + _, err := a.performRequest(ctx, "PUT", l, r) + if err != nil { + return err + } + return nil +} + +// Verify calls the verify handler on the LFS server +func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error { + b, err := jsoniter.Marshal(p) + if err != nil { + return fmt.Errorf("lfs.BasicTransferAdapter.Verify json.Marshal: %w", err) + } + + _, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b)) + if err != nil { + return err + } + return nil +} + +func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, r io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, l.Href, r) + if err != nil { + return nil, fmt.Errorf("lfs.BasicTransferAdapter.performRequest http.NewRequestWithContext: %w", err) } for key, value := range l.Header { req.Header.Set(key, value) @@ -42,11 +78,31 @@ func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCl if err != nil { select { case <-ctx.Done(): - return nil, ctx.Err() + return res, ctx.Err() default: } - return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.Do: %w", err) + return res, fmt.Errorf("lfs.BasicTransferAdapter.performRequest http.Do: %w", err) + } + + if res.StatusCode != http.StatusOK { + return res, handleErrorResponse(res) + } + + return res, nil +} + +func handleErrorResponse(resp *http.Response) error { + defer resp.Body.Close() + + er, err := decodeReponseError(resp.Body) + if err != nil { + return fmt.Errorf("Request failed with status %s", resp.Status) } + return errors.New(er.Message) +} - return res.Body, nil +func decodeReponseError(r io.Reader) (ErrorResponse, error) { + var er ErrorResponse + err := jsoniter.NewDecoder(r).Decode(&er) + return er, err } diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index 0eabd3faeee1..69df24ad6b67 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -40,35 +40,21 @@ func TestBasicTransferAdapterDownload(t *testing.T) { a := &BasicTransferAdapter{hc} var cases = []struct { - response *ObjectResponse + link *Link expectederror string }{ // case 0 { - response: &ObjectResponse{}, - expectederror: "Action 'download' not found", - }, - // case 1 - { - response: &ObjectResponse{ - Actions: map[string]*Link{"upload": nil}, - }, - expectederror: "Action 'download' not found", - }, - // case 2 - { - response: &ObjectResponse{ - Actions: map[string]*Link{"download": { - Href: "https://valid-download-request.io", - Header: map[string]string{"test-header": "test-value"}, - }}, + link: &Link{ + Href: "https://valid-download-request.io", + Header: map[string]string{"test-header": "test-value"}, }, expectederror: "", }, } for n, c := range cases { - _, err := a.Download(context.Background(), c.response) + _, err := a.Download(context.Background(), c.link) if len(c.expectederror) > 0 { assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { From 92e2573d347870c7a57b92dc8f3b03e1c54d1dea Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 18 Apr 2021 16:23:54 +0000 Subject: [PATCH 24/54] Added "upload" unit tests. --- modules/lfs/http_client.go | 4 +- modules/lfs/http_client_test.go | 290 +++++++++++++++++++++++++++----- 2 files changed, 247 insertions(+), 47 deletions(-) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index abfacdce9c7f..fd969ff7bc2c 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -147,11 +147,11 @@ func (c *HTTPClient) performOperation(ctx context.Context, bc *batchContext) err transferAdapter, ok := c.transfers[result.Transfer] if !ok { - return fmt.Errorf("LFS TransferAdapter not found: %s", result.Transfer) + return fmt.Errorf("TransferAdapter not found: %s", result.Transfer) } if len(result.Objects) == 0 { - return errors.New("No LFS objects in result") + return errors.New("No objects in result") } object := result.Objects[0] diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index b2b684700f55..d5d3743d1f36 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -42,64 +42,233 @@ func (a *DummyTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) e return nil } -func TestHTTPClientDownload(t *testing.T) { - p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} +func lfsTestRoundtripHandler(req *http.Request) *http.Response { + var batchResponse *BatchResponse + url := req.URL.String() - roundTripHandler := func(req *http.Request) *http.Response { - url := req.URL.String() - if strings.Contains(url, "status-not-ok") { - return &http.Response{StatusCode: http.StatusBadRequest} + if strings.Contains(url, "status-not-ok") { + return &http.Response{StatusCode: http.StatusBadRequest} + } else if strings.Contains(url, "invalid-json-response") { + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} + } else if strings.Contains(url, "valid-batch-request-download") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Actions: map[string]*Link{ + "download": &Link{}, + }, + }, + }, + } + } else if strings.Contains(url, "valid-batch-request-upload") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Actions: map[string]*Link{ + "upload": &Link{}, + }, + }, + }, + } + } else if strings.Contains(url, "invalid-response-no-objects") { + batchResponse = &BatchResponse{Transfer: "dummy"} + } else if strings.Contains(url, "unknown-transfer-adapter") { + batchResponse = &BatchResponse{Transfer: "unknown_adapter"} + } else if strings.Contains(url, "error-in-response-objects") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Error: &ObjectError{ + Code: 404, + Message: "Object not found", + }, + }, + }, + } + } else if strings.Contains(url, "empty-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Actions: map[string]*Link{}, + }, + }, + } + } else if strings.Contains(url, "download-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Actions: map[string]*Link{ + "download": &Link{}, + }, + }, + }, } - if strings.Contains(url, "invalid-json-response") { - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} + } else if strings.Contains(url, "upload-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Actions: map[string]*Link{ + "upload": &Link{}, + }, + }, + }, } - if strings.Contains(url, "valid-batch-request-download") { - assert.Equal(t, "POST", req.Method) - assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url) - assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url) + } else if strings.Contains(url, "verify-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Actions: map[string]*Link{ + "verify": &Link{}, + }, + }, + }, + } + } else if strings.Contains(url, "unknown-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + &ObjectResponse{ + Actions: map[string]*Link{ + "unknown": &Link{}, + }, + }, + }, + } + } else { + return nil + } + + payload := new(bytes.Buffer) + json.NewEncoder(payload).Encode(batchResponse) - var batchRequest BatchRequest - err := json.NewDecoder(req.Body).Decode(&batchRequest) - assert.NoError(t, err) + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} +} - assert.Equal(t, "download", batchRequest.Operation) - assert.Equal(t, 1, len(batchRequest.Objects)) - assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid) - assert.Equal(t, p.Size, batchRequest.Objects[0].Size) +func TestHTTPClientDownload(t *testing.T) { + p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} - batchResponse := &BatchResponse{ - Transfer: "dummy", - Objects: make([]*ObjectResponse, 1), - } + hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response { + assert.Equal(t, "POST", req.Method) + assert.Equal(t, MediaType, req.Header.Get("Content-type")) + assert.Equal(t, MediaType, req.Header.Get("Accept")) - payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(batchResponse) + var batchRequest BatchRequest + err := json.NewDecoder(req.Body).Decode(&batchRequest) + assert.NoError(t, err) - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} - } - if strings.Contains(url, "invalid-response-no-objects") { - batchResponse := &BatchResponse{Transfer: "dummy"} + assert.Equal(t, "download", batchRequest.Operation) + assert.Equal(t, 1, len(batchRequest.Objects)) + assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid) + assert.Equal(t, p.Size, batchRequest.Objects[0].Size) - payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(batchResponse) + return lfsTestRoundtripHandler(req) + })} + dummy := &DummyTransferAdapter{} - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} - } - if strings.Contains(url, "unknown-transfer-adapter") { - batchResponse := &BatchResponse{Transfer: "unknown_adapter"} + var cases = []struct { + endpoint string + expectederror string + }{ + // case 0 + { + endpoint: "https://status-not-ok.io", + expectederror: "Unexpected servers response: ", + }, + // case 1 + { + endpoint: "https://invalid-json-response.io", + expectederror: "json.Decode: ", + }, + // case 2 + { + endpoint: "https://valid-batch-request-download.io", + expectederror: "", + }, + // case 3 + { + endpoint: "https://invalid-response-no-objects.io", + expectederror: "No objects in result", + }, + // case 4 + { + endpoint: "https://unknown-transfer-adapter.io", + expectederror: "TransferAdapter not found: ", + }, + // case 5 + { + endpoint: "https://error-in-response-objects.io", + expectederror: "Object not found", + }, + // case 6 + { + endpoint: "https://empty-actions-map.io", + expectederror: "Action 'download' not found", + }, + // case 7 + { + endpoint: "https://download-actions-map.io", + expectederror: "", + }, + // case 8 + { + endpoint: "https://upload-actions-map.io", + expectederror: "Action 'download' not found", + }, + // case 9 + { + endpoint: "https://verify-actions-map.io", + expectederror: "Action 'download' not found", + }, + // case 10 + { + endpoint: "https://unknown-actions-map.io", + expectederror: "Action 'download' not found", + }, + } - payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(batchResponse) + for n, c := range cases { + client := &HTTPClient{ + client: hc, + endpoint: c.endpoint, + transfers: make(map[string]TransferAdapter), + } + client.transfers["dummy"] = dummy - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} + _, err := client.Download(context.Background(), p) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) } + } +} + +func TestHTTPClientUpload(t *testing.T) { + p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} - t.Errorf("Unknown test case: %s", url) + hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response { + assert.Equal(t, "POST", req.Method) + assert.Equal(t, MediaType, req.Header.Get("Content-type")) + assert.Equal(t, MediaType, req.Header.Get("Accept")) - return nil - } + var batchRequest BatchRequest + err := json.NewDecoder(req.Body).Decode(&batchRequest) + assert.NoError(t, err) - hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} + assert.Equal(t, "upload", batchRequest.Operation) + assert.Equal(t, 1, len(batchRequest.Objects)) + assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid) + assert.Equal(t, p.Size, batchRequest.Objects[0].Size) + + return lfsTestRoundtripHandler(req) + })} dummy := &DummyTransferAdapter{} var cases = []struct { @@ -118,7 +287,7 @@ func TestHTTPClientDownload(t *testing.T) { }, // case 2 { - endpoint: "https://valid-batch-request-download.io", + endpoint: "https://valid-batch-request-upload.io", expectederror: "", }, // case 3 @@ -129,10 +298,41 @@ func TestHTTPClientDownload(t *testing.T) { // case 4 { endpoint: "https://unknown-transfer-adapter.io", - expectederror: "Transferadapter not found: ", + expectederror: "TransferAdapter not found: ", + }, + // case 5 + { + endpoint: "https://error-in-response-objects.io", + expectederror: "Object not found", + }, + // case 6 + { + endpoint: "https://empty-actions-map.io", + expectederror: "", + }, + // case 7 + { + endpoint: "https://download-actions-map.io", + expectederror: "Action 'upload' not found", + }, + // case 8 + { + endpoint: "https://upload-actions-map.io", + expectederror: "", + }, + // case 9 + { + endpoint: "https://verify-actions-map.io", + expectederror: "Action 'upload' not found", + }, + // case 10 + { + endpoint: "https://unknown-actions-map.io", + expectederror: "Action 'upload' not found", }, } + r := new(bytes.Buffer) for n, c := range cases { client := &HTTPClient{ client: hc, @@ -141,7 +341,7 @@ func TestHTTPClientDownload(t *testing.T) { } client.transfers["dummy"] = dummy - _, err := client.Download(context.Background(), p) + err := client.Upload(context.Background(), p, r) if len(c.expectederror) > 0 { assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { From 0d9c384d24806e8e9187dd4b391f55a582b72016 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 18 Apr 2021 17:10:06 +0000 Subject: [PATCH 25/54] Added transfer adapter unit tests. --- modules/lfs/http_client_test.go | 46 ++++---- modules/lfs/shared.go | 2 +- modules/lfs/transferadapter_test.go | 156 +++++++++++++++++++++++----- 3 files changed, 155 insertions(+), 49 deletions(-) diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index d5d3743d1f36..a5039831e763 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -53,10 +53,10 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "valid-batch-request-download") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Actions: map[string]*Link{ - "download": &Link{}, + "download": {}, }, }, }, @@ -64,10 +64,10 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "valid-batch-request-upload") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Actions: map[string]*Link{ - "upload": &Link{}, + "upload": {}, }, }, }, @@ -79,10 +79,10 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "error-in-response-objects") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Error: &ObjectError{ - Code: 404, + Code: 404, Message: "Object not found", }, }, @@ -91,8 +91,8 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "empty-actions-map") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Actions: map[string]*Link{}, }, }, @@ -100,10 +100,10 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "download-actions-map") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Actions: map[string]*Link{ - "download": &Link{}, + "download": {}, }, }, }, @@ -111,10 +111,10 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "upload-actions-map") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Actions: map[string]*Link{ - "upload": &Link{}, + "upload": {}, }, }, }, @@ -122,10 +122,10 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "verify-actions-map") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Actions: map[string]*Link{ - "verify": &Link{}, + "verify": {}, }, }, }, @@ -133,10 +133,10 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } else if strings.Contains(url, "unknown-actions-map") { batchResponse = &BatchResponse{ Transfer: "dummy", - Objects: []*ObjectResponse{ - &ObjectResponse{ + Objects: []*ObjectResponse{ + { Actions: map[string]*Link{ - "unknown": &Link{}, + "unknown": {}, }, }, }, diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 8a5ea2abe34e..987ad643689b 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -71,6 +71,6 @@ type PointerBlob struct { // ErrorResponse describes the error to the client. type ErrorResponse struct { Message string - DocumentationUrl string `json:"documentation_url"` + DocumentationURL string `json:"documentation_url"` RequestID string `json:"request_id"` } diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index 69df24ad6b67..b125edb633fc 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -7,6 +7,8 @@ package lfs import ( "bytes" "context" + "encoding/json" + "io" "io/ioutil" "net/http" "strings" @@ -21,44 +23,148 @@ func TestBasicTransferAdapterName(t *testing.T) { assert.Equal(t, "basic", a.Name()) } -func TestBasicTransferAdapterDownload(t *testing.T) { +func TestBasicTransferAdapter(t *testing.T) { + p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} + roundTripHandler := func(req *http.Request) *http.Response { + assert.Equal(t, "test-value", req.Header.Get("test-header")) + url := req.URL.String() - if strings.Contains(url, "valid-download-request") { + if strings.Contains(url, "download-request") { assert.Equal(t, "GET", req.Method) - assert.Equal(t, "test-value", req.Header.Get("test-header")) return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))} - } + } else if strings.Contains(url, "upload-request") { + assert.Equal(t, "PUT", req.Method) + + b, err := io.ReadAll(req.Body) + assert.NoError(t, err) + assert.Equal(t, "dummy", string(b)) - t.Errorf("Unknown test case: %s", url) + return &http.Response{StatusCode: http.StatusOK} + } else if strings.Contains(url, "verify-request") { + assert.Equal(t, "POST", req.Method) - return nil + var vp Pointer + err := json.NewDecoder(req.Body).Decode(&vp) + assert.NoError(t, err) + assert.Equal(t, p.Oid, vp.Oid) + assert.Equal(t, p.Size, vp.Size) + + return &http.Response{StatusCode: http.StatusOK} + } else if strings.Contains(url, "error-response") { + er := &ErrorResponse{ + Message: "Object not found", + } + payload := new(bytes.Buffer) + json.NewEncoder(payload).Encode(er) + + return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(payload)} + } else { + t.Errorf("Unknown test case: %s", url) + return nil + } } hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} a := &BasicTransferAdapter{hc} - var cases = []struct { - link *Link - expectederror string - }{ - // case 0 - { - link: &Link{ - Href: "https://valid-download-request.io", - Header: map[string]string{"test-header": "test-value"}, + t.Run("Download", func(t *testing.T) { + cases := []struct { + link *Link + expectederror string + }{ + // case 0 + { + link: &Link{ + Href: "https://download-request.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "", }, - expectederror: "", - }, - } + // case 1 + { + link: &Link{ + Href: "https://error-response.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "Object not found", + }, + } - for n, c := range cases { - _, err := a.Download(context.Background(), c.link) - if len(c.expectederror) > 0 { - assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) - } else { - assert.NoError(t, err, "case %d", n) + for n, c := range cases { + _, err := a.Download(context.Background(), c.link) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } } - } + }) + + t.Run("Upload", func(t *testing.T) { + cases := []struct { + link *Link + expectederror string + }{ + // case 0 + { + link: &Link{ + Href: "https://upload-request.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "", + }, + // case 1 + { + link: &Link{ + Href: "https://error-response.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "Object not found", + }, + } + + for n, c := range cases { + err := a.Upload(context.Background(), c.link, bytes.NewBuffer([]byte("dummy"))) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } + } + }) + + t.Run("Verify", func(t *testing.T) { + cases := []struct { + link *Link + expectederror string + }{ + // case 0 + { + link: &Link{ + Href: "https://verify-request.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "", + }, + // case 1 + { + link: &Link{ + Href: "https://error-response.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "Object not found", + }, + } + + for n, c := range cases { + err := a.Verify(context.Background(), c.link, p) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } + } + }) } From 3bc230079bc39586f06653bc1031b80004085b93 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 18 Apr 2021 18:30:09 +0000 Subject: [PATCH 26/54] Send correct headers. --- modules/lfs/transferadapter.go | 3 +++ modules/lfs/transferadapter_test.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 6a35d7b9e0af..df364648ebb8 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -58,6 +58,8 @@ func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) e return fmt.Errorf("lfs.BasicTransferAdapter.Verify json.Marshal: %w", err) } + l.Header["Content-Type"] = MediaType + _, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b)) if err != nil { return err @@ -73,6 +75,7 @@ func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string for key, value := range l.Header { req.Header.Set(key, value) } + req.Header.Set("Accept", MediaType) res, err := a.client.Do(req) if err != nil { diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index b125edb633fc..13b5b1138a4c 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -27,6 +27,7 @@ func TestBasicTransferAdapter(t *testing.T) { p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} roundTripHandler := func(req *http.Request) *http.Response { + assert.Equal(t, MediaType, req.Header.Get("Accept")) assert.Equal(t, "test-value", req.Header.Get("test-header")) url := req.URL.String() @@ -44,6 +45,7 @@ func TestBasicTransferAdapter(t *testing.T) { return &http.Response{StatusCode: http.StatusOK} } else if strings.Contains(url, "verify-request") { assert.Equal(t, "POST", req.Method) + assert.Equal(t, MediaType, req.Header.Get("Content-Type")) var vp Pointer err := json.NewDecoder(req.Body).Decode(&vp) From c4ff5d2ef5ae84d6b24a68f4bb3bd21534265cd8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 19 Apr 2021 11:43:09 +0000 Subject: [PATCH 27/54] Added pushing of LFS objects. --- modules/lfs/client_test.go | 1 - modules/lfs/http_client.go | 4 +- modules/lfs/http_client_test.go | 2 +- modules/lfs/transferadapter.go | 56 +++++++++++++++---- modules/lfs/transferadapter_test.go | 6 ++- services/mirror/mirror_pull.go | 2 +- services/mirror/mirror_push.go | 84 ++++++++++++++++++++++++++--- 7 files changed, 130 insertions(+), 25 deletions(-) diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go index d4eb00546948..1040b3992560 100644 --- a/modules/lfs/client_test.go +++ b/modules/lfs/client_test.go @@ -6,7 +6,6 @@ package lfs import ( "net/url" - "testing" "github.com/stretchr/testify/assert" diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index fd969ff7bc2c..f0e747bb5143 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -66,7 +66,7 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err) } - log.Trace("lfs.HTTPClient.batch NewRequestWithContext: %s", url) + log.Trace("lfs.HTTPClient.batch calling: %s", url) req, err := http.NewRequestWithContext(ctx, "POST", url, payload) if err != nil { @@ -170,7 +170,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, bc *batchContext) err return errors.New("Action 'upload' not found") } - if err := transferAdapter.Upload(ctx, link, bc.UploadContent); err != nil { + if err := transferAdapter.Upload(ctx, link, bc.Pointer, bc.UploadContent); err != nil { return err } diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index a5039831e763..0247e2d329f6 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -34,7 +34,7 @@ func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCl return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil } -func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, r io.Reader) error { +func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { return nil } diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index df364648ebb8..9109abec19bd 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -11,6 +11,9 @@ import ( "fmt" "io" "net/http" + "strconv" + + "code.gitea.io/gitea/modules/log" jsoniter "github.com/json-iterator/go" ) @@ -19,7 +22,7 @@ import ( type TransferAdapter interface { Name() string Download(ctx context.Context, l *Link) (io.ReadCloser, error) - Upload(ctx context.Context, l *Link, r io.Reader) error + Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error Verify(ctx context.Context, l *Link, p Pointer) error } @@ -43,8 +46,23 @@ func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCl } // Upload sends the content to the LFS server -func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, r io.Reader) error { - _, err := a.performRequest(ctx, "PUT", l, r) +func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { + _, err := a.performRequest(ctx, "PUT", l, func(req *http.Request) error { + if len(req.Header.Get("Content-Type")) == 0 { + req.Header.Set("Content-Type", "application/octet-stream") + } + + if req.Header.Get("Transfer-Encoding") == "chunked" { + req.TransferEncoding = []string{"chunked"} + } else { + req.Header.Set("Content-Length", strconv.FormatInt(p.Size, 10)) + } + + req.Body = io.NopCloser(r) + req.ContentLength = p.Size + + return nil + }) if err != nil { return err } @@ -53,22 +71,32 @@ func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, r io.Reader) // Verify calls the verify handler on the LFS server func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error { - b, err := jsoniter.Marshal(p) - if err != nil { - return fmt.Errorf("lfs.BasicTransferAdapter.Verify json.Marshal: %w", err) - } + _, err := a.performRequest(ctx, "POST", l, func(req *http.Request) error { + b, err := jsoniter.Marshal(p) + if err != nil { + return fmt.Errorf("lfs.BasicTransferAdapter.Verify json.Marshal: %w", err) + } + + size := int64(len(b)) - l.Header["Content-Type"] = MediaType + req.Header.Set("Content-Type", MediaType) + req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) - _, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b)) + req.Body = io.NopCloser(bytes.NewReader(b)) + req.ContentLength = size + + return nil + }) if err != nil { return err } return nil } -func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, r io.Reader) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, method, l.Href, r) +func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, rcb func(*http.Request) error) (*http.Response, error) { + log.Trace("lfs.BasicTransferAdapter.performRequest calling: %s %s", method, l.Href) + + req, err := http.NewRequestWithContext(ctx, method, l.Href, nil) if err != nil { return nil, fmt.Errorf("lfs.BasicTransferAdapter.performRequest http.NewRequestWithContext: %w", err) } @@ -77,6 +105,12 @@ func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string } req.Header.Set("Accept", MediaType) + if rcb != nil { + if err := rcb(req); err != nil { + return nil, err + } + } + res, err := a.client.Do(req) if err != nil { select { diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index 13b5b1138a4c..a02d0a2e3ab7 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -24,7 +24,7 @@ func TestBasicTransferAdapterName(t *testing.T) { } func TestBasicTransferAdapter(t *testing.T) { - p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} + p := Pointer{Oid: "b5a2c96250612366ea272ffac6d9744aaf4b45aacd96aa7cfcb931ee3b558259", Size: 5} roundTripHandler := func(req *http.Request) *http.Response { assert.Equal(t, MediaType, req.Header.Get("Accept")) @@ -37,6 +37,8 @@ func TestBasicTransferAdapter(t *testing.T) { return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))} } else if strings.Contains(url, "upload-request") { assert.Equal(t, "PUT", req.Method) + assert.Equal(t, "application/octet-stream", req.Header.Get("Content-Type")) + assert.Equal(t, "5", req.Header.Get("Content-Length")) b, err := io.ReadAll(req.Body) assert.NoError(t, err) @@ -128,7 +130,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - err := a.Upload(context.Background(), c.link, bytes.NewBuffer([]byte("dummy"))) + err := a.Upload(context.Background(), c.link, p, bytes.NewBufferString("dummy")) if len(c.expectederror) > 0 { assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index db65a848bacc..9b77d05c3431 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -193,7 +193,6 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) } - gitRepo.Close() if m.LFS && setting.LFS.StartServer { log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) @@ -202,6 +201,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) log.Error("Failed to synchronize LFS objects for repository: %v", err) } } + gitRepo.Close() log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 3154886eabec..cb63c67b3a8e 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -7,11 +7,14 @@ package mirror import ( "context" "errors" + "fmt" + "net/url" "strconv" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -87,10 +90,12 @@ func syncPushMirror(ctx context.Context, mirrorID string) { } m.UpdatedUnix = timeutil.TimeStampNow() + m.LastError = "" log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Running Sync", mirrorID, m.Repo) err = runPushSync(ctx, m) if err != nil { + log.Error("SyncPushMirror [mirror: %s][repo: %-v]: %v", err) m.LastError = err.Error() } @@ -109,6 +114,28 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { performPush := func(path string) error { log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + remoteAddr, err := git.GetRemoteAddress(path, m.RemoteName) + if err != nil { + log.Error("GetRemoteAddress(%s) Error %v", path, err) + return errors.New("Unexpected error") + } + + if setting.LFS.StartServer { + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + + gitRepo, err := git.OpenRepository(path) + if err != nil { + log.Error("OpenRepository: %v", err) + return errors.New("Unexpected error") + } + defer gitRepo.Close() + + ep := lfs.DetermineEndpoint(remoteAddr.String(), "") + if err := pushAllLFSObjects(ctx, gitRepo, ep); err != nil { + return util.NewURLSanitizedError(err, remoteAddr, true) + } + } + if err := git.Push(path, git.PushOptions{ Remote: m.RemoteName, Force: true, @@ -117,13 +144,9 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { }); err != nil { log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) - remoteAddr, remoteErr := git.GetRemoteAddress(path, m.RemoteName) - if remoteErr != nil { - log.Error("GetRemoteAddress(%s) Error %v", path, remoteErr) - return errors.New("Unexpected error") - } return util.NewURLSanitizedError(err, remoteAddr, true) } + return nil } @@ -132,8 +155,6 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { return err } - // TODO LFS - if m.Repo.HasWiki() { err := performPush(m.Repo.WikiPath()) if err != nil { @@ -143,3 +164,52 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { return nil } + +func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL) error { + client := lfs.NewClient(endpoint) + contentStore := lfs.NewContentStore() + + pointerChan := make(chan lfs.PointerBlob) + errChan := make(chan error, 1) + go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) + + err := func() error { + for pointerBlob := range pointerChan { + err := func() error { + if exists, err := contentStore.Exists(pointerBlob.Pointer); !exists || err != nil { + return err + } + + content, err := contentStore.Get(pointerBlob.Pointer) + if err != nil { + return err + } + defer content.Close() + + if err := client.Upload(ctx, pointerBlob.Pointer, content); err != nil { + return fmt.Errorf("pushAllLFSObjects: LFS OID[%s] failed to upload: %w", pointerBlob.Oid, err) + } + return nil + }() + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + return err + } + } + return nil + }() + if err != nil { + return err + } + + err, has := <-errChan + if has { + return err + } + + return nil +} From 0012f347f6231a96e780d4fcbc1fd156e3af44be Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 19 Apr 2021 22:52:26 +0000 Subject: [PATCH 28/54] Go 1.14 --- modules/lfs/transferadapter.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 9109abec19bd..94f6100618cc 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "net/http" "strconv" @@ -58,7 +59,11 @@ func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r req.Header.Set("Content-Length", strconv.FormatInt(p.Size, 10)) } - req.Body = io.NopCloser(r) + rc, ok := r.(io.ReadCloser) + if !ok && r != nil { + rc = ioutil.NopCloser(r) + } + req.Body = rc req.ContentLength = p.Size return nil @@ -82,7 +87,7 @@ func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) e req.Header.Set("Content-Type", MediaType) req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) - req.Body = io.NopCloser(bytes.NewReader(b)) + req.Body = ioutil.NopCloser(bytes.NewReader(b)) req.ContentLength = size return nil From 0b3e91abf078409c4203f0ceff5601c330ffa907 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Apr 2021 10:59:51 +0000 Subject: [PATCH 29/54] Simpler body handling. --- modules/lfs/transferadapter.go | 33 ++++++++--------------------- modules/lfs/transferadapter_test.go | 1 - 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 94f6100618cc..9f86118321c9 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -10,9 +10,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" - "strconv" "code.gitea.io/gitea/modules/log" @@ -39,7 +37,7 @@ func (a *BasicTransferAdapter) Name() string { // Download reads the download location and downloads the data func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { - resp, err := a.performRequest(ctx, "GET", l, nil) + resp, err := a.performRequest(ctx, "GET", l, nil, nil) if err != nil { return nil, err } @@ -48,22 +46,15 @@ func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCl // Upload sends the content to the LFS server func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { - _, err := a.performRequest(ctx, "PUT", l, func(req *http.Request) error { + _, err := a.performRequest(ctx, "PUT", l, r, func(req *http.Request) error { if len(req.Header.Get("Content-Type")) == 0 { req.Header.Set("Content-Type", "application/octet-stream") } if req.Header.Get("Transfer-Encoding") == "chunked" { req.TransferEncoding = []string{"chunked"} - } else { - req.Header.Set("Content-Length", strconv.FormatInt(p.Size, 10)) } - rc, ok := r.(io.ReadCloser) - if !ok && r != nil { - rc = ioutil.NopCloser(r) - } - req.Body = rc req.ContentLength = p.Size return nil @@ -76,19 +67,13 @@ func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r // Verify calls the verify handler on the LFS server func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error { - _, err := a.performRequest(ctx, "POST", l, func(req *http.Request) error { - b, err := jsoniter.Marshal(p) - if err != nil { - return fmt.Errorf("lfs.BasicTransferAdapter.Verify json.Marshal: %w", err) - } - - size := int64(len(b)) + b, err := jsoniter.Marshal(p) + if err != nil { + return fmt.Errorf("lfs.BasicTransferAdapter.Verify json.Marshal: %w", err) + } + _, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b), func(req *http.Request) error { req.Header.Set("Content-Type", MediaType) - req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) - - req.Body = ioutil.NopCloser(bytes.NewReader(b)) - req.ContentLength = size return nil }) @@ -98,10 +83,10 @@ func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) e return nil } -func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, rcb func(*http.Request) error) (*http.Response, error) { +func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, body io.Reader, rcb func(*http.Request) error) (*http.Response, error) { log.Trace("lfs.BasicTransferAdapter.performRequest calling: %s %s", method, l.Href) - req, err := http.NewRequestWithContext(ctx, method, l.Href, nil) + req, err := http.NewRequestWithContext(ctx, method, l.Href, body) if err != nil { return nil, fmt.Errorf("lfs.BasicTransferAdapter.performRequest http.NewRequestWithContext: %w", err) } diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index a02d0a2e3ab7..e409e095c067 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -38,7 +38,6 @@ func TestBasicTransferAdapter(t *testing.T) { } else if strings.Contains(url, "upload-request") { assert.Equal(t, "PUT", req.Method) assert.Equal(t, "application/octet-stream", req.Header.Get("Content-Type")) - assert.Equal(t, "5", req.Header.Get("Content-Length")) b, err := io.ReadAll(req.Body) assert.NoError(t, err) From f2cbd62d68a7340c51992e1172cd1cc97f7137b9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Apr 2021 12:13:00 +0200 Subject: [PATCH 30/54] Process files in batches to reduce HTTP calls. --- modules/lfs/client.go | 11 ++- modules/lfs/filesystem_client.go | 62 +++++++++++---- modules/lfs/http_client.go | 128 +++++++++++++++++-------------- modules/lfs/http_client_test.go | 25 ++++-- modules/repository/repo.go | 105 ++++++++++++++----------- services/mirror/mirror_push.go | 65 +++++++++------- 6 files changed, 244 insertions(+), 152 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index fb1a843f0a08..0a21440f73d0 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -10,10 +10,17 @@ import ( "net/url" ) +// DownloadCallback gets called for every requested LFS object to process its content +type DownloadCallback func(p Pointer, content io.ReadCloser, objectError error) error + +// UploadCallback gets called for every requested LFS object to provide its content +type UploadCallback func(p Pointer, objectError error) (io.ReadCloser, error) + // Client is used to communicate with a LFS source type Client interface { - Download(ctx context.Context, p Pointer) (io.ReadCloser, error) - Upload(ctx context.Context, p Pointer, content io.Reader) error + BatchSize() int + Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error + Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error } // NewClient creates a LFS client diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 9ba8b206aafe..dc72981a9ec9 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -19,6 +19,11 @@ type FilesystemClient struct { lfsdir string } +// BatchSize returns the preferred size of batchs to process +func (c *FilesystemClient) BatchSize() int { + return 1 +} + func newFilesystemClient(endpoint *url.URL) *FilesystemClient { path, _ := util.FileURLToPath(endpoint) @@ -34,26 +39,55 @@ func (c *FilesystemClient) objectPath(oid string) string { } // Download reads the specific LFS object from the target path -func (c *FilesystemClient) Download(ctx context.Context, p Pointer) (io.ReadCloser, error) { - objectPath := c.objectPath(p.Oid) +func (c *FilesystemClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { + for _, object := range objects { + p := Pointer{object.Oid, object.Size} + + objectPath := c.objectPath(p.Oid) - return os.Open(objectPath) + f, err := os.Open(objectPath) + if err != nil { + return err + } + + if err := callback(p, f, nil); err != nil { + return err + } + } + return nil } // Upload writes the specific LFS object to the target path -func (c *FilesystemClient) Upload(ctx context.Context, p Pointer, r io.Reader) error { - objectPath := c.objectPath(p.Oid) +func (c *FilesystemClient) Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error { + for _, object := range objects { + p := Pointer{object.Oid, object.Size} - if err := os.MkdirAll(filepath.Dir(objectPath), 0600); err != nil { - return err - } + objectPath := c.objectPath(p.Oid) - f, err := os.Create(objectPath) - if err != nil { - return err - } + if err := os.MkdirAll(filepath.Dir(objectPath), os.ModePerm); err != nil { + return err + } + + content, err := callback(p, nil) + if err != nil { + return err + } + + err = func() error { + defer content.Close() - _, err = io.Copy(f, r) + f, err := os.Create(objectPath) + if err != nil { + return err + } - return err + _, err = io.Copy(f, content) + + return err + }() + if err != nil { + return err + } + } + return nil } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index f0e747bb5143..11fd4ed7f08b 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -9,7 +9,6 @@ import ( "context" "errors" "fmt" - "io" "net/http" "net/url" "strings" @@ -19,6 +18,8 @@ import ( jsoniter "github.com/json-iterator/go" ) +const batchSize = 20 + // HTTPClient is used to communicate with the LFS server // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md type HTTPClient struct { @@ -27,6 +28,11 @@ type HTTPClient struct { transfers map[string]TransferAdapter } +// BatchSize returns the preferred size of batchs to process +func (c *HTTPClient) BatchSize() int { + return batchSize +} + func newHTTPClient(endpoint *url.URL) *HTTPClient { hc := &http.Client{} @@ -104,43 +110,26 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin } // Download reads the specific LFS object from the LFS server -func (c *HTTPClient) Download(ctx context.Context, p Pointer) (io.ReadCloser, error) { - bc := batchContext{ - IsUpload: false, - Pointer: p, - } - err := c.performOperation(ctx, &bc) - if err != nil { - return nil, err - } - return bc.DownloadResult, nil +func (c *HTTPClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { + return c.performOperation(ctx, objects, callback, nil) } // Upload sends the specific LFS object to the LFS server -func (c *HTTPClient) Upload(ctx context.Context, p Pointer, r io.Reader) error { - bc := batchContext{ - IsUpload: true, - Pointer: p, - UploadContent: r, - } - return c.performOperation(ctx, &bc) +func (c *HTTPClient) Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error { + return c.performOperation(ctx, objects, nil, callback) } -type batchContext struct { - Pointer - IsUpload bool - - DownloadResult io.ReadCloser - UploadContent io.Reader -} +func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc DownloadCallback, uc UploadCallback) error { + if len(objects) == 0 { + return nil + } -func (c *HTTPClient) performOperation(ctx context.Context, bc *batchContext) error { operation := "download" - if bc.IsUpload { + if uc != nil { operation = "upload" } - result, err := c.batch(ctx, operation, []Pointer{bc.Pointer}) + result, err := c.batch(ctx, operation, objects) if err != nil { return err } @@ -150,47 +139,72 @@ func (c *HTTPClient) performOperation(ctx context.Context, bc *batchContext) err return fmt.Errorf("TransferAdapter not found: %s", result.Transfer) } - if len(result.Objects) == 0 { - return errors.New("No objects in result") - } + if uc != nil { + for _, object := range result.Objects { + p := Pointer{object.Oid, object.Size} - object := result.Objects[0] + if object.Error != nil { + if _, err := uc(p, errors.New(object.Error.Message)); err != nil { + return err + } + continue + } - if object.Error != nil { - return errors.New(object.Error.Message) - } + if len(object.Actions) == 0 { + continue + } - if bc.IsUpload { - if len(object.Actions) == 0 { - return nil - } + link, ok := object.Actions["upload"] + if !ok { + return errors.New("Action 'upload' not found") + } - link, ok := object.Actions["upload"] - if !ok { - return errors.New("Action 'upload' not found") - } + content, err := uc(p, nil) + if err != nil { + return err + } - if err := transferAdapter.Upload(ctx, link, bc.Pointer, bc.UploadContent); err != nil { - return err - } + err = transferAdapter.Upload(ctx, link, p, content) + + content.Close() - link, ok = object.Actions["verify"] - if ok { - if err := transferAdapter.Verify(ctx, link, bc.Pointer); err != nil { + if err != nil { return err } + + link, ok = object.Actions["verify"] + if ok { + if err := transferAdapter.Verify(ctx, link, p); err != nil { + return err + } + } } } else { - link, ok := object.Actions["download"] - if !ok { - return errors.New("Action 'download' not found") - } + for _, object := range result.Objects { + p := Pointer{object.Oid, object.Size} + + if object.Error != nil { + if err := dc(p, nil, errors.New(object.Error.Message)); err != nil { + return err + } + continue + } - var err error - bc.DownloadResult, err = transferAdapter.Download(ctx, link) - if err != nil { - return err + link, ok := object.Actions["download"] + if !ok { + return errors.New("Action 'download' not found") + } + + content, err := transferAdapter.Download(ctx, link) + if err != nil { + return err + } + + if err := dc(p, content, nil); err != nil { + return err + } } } + return nil } diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 0247e2d329f6..e2cc8f1649ee 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -72,7 +72,7 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { }, }, } - } else if strings.Contains(url, "invalid-response-no-objects") { + } else if strings.Contains(url, "response-no-objects") { batchResponse = &BatchResponse{Transfer: "dummy"} } else if strings.Contains(url, "unknown-transfer-adapter") { batchResponse = &BatchResponse{Transfer: "unknown_adapter"} @@ -193,8 +193,8 @@ func TestHTTPClientDownload(t *testing.T) { }, // case 3 { - endpoint: "https://invalid-response-no-objects.io", - expectederror: "No objects in result", + endpoint: "https://response-no-objects.io", + expectederror: "", }, // case 4 { @@ -241,7 +241,15 @@ func TestHTTPClientDownload(t *testing.T) { } client.transfers["dummy"] = dummy - _, err := client.Download(context.Background(), p) + err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { + if objectError != nil { + return objectError + } + b, err := io.ReadAll(content) + assert.NoError(t, err) + assert.Equal(t, []byte("dummy"), b) + return nil + }) if len(c.expectederror) > 0 { assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { @@ -292,8 +300,8 @@ func TestHTTPClientUpload(t *testing.T) { }, // case 3 { - endpoint: "https://invalid-response-no-objects.io", - expectederror: "No objects in result", + endpoint: "https://response-no-objects.io", + expectederror: "", }, // case 4 { @@ -332,7 +340,6 @@ func TestHTTPClientUpload(t *testing.T) { }, } - r := new(bytes.Buffer) for n, c := range cases { client := &HTTPClient{ client: hc, @@ -341,7 +348,9 @@ func TestHTTPClientUpload(t *testing.T) { } client.transfers["dummy"] = dummy - err := client.Upload(context.Background(), p, r) + err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) { + return ioutil.NopCloser(new(bytes.Buffer)), objectError + }) if len(c.expectederror) > 0 { assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { diff --git a/modules/repository/repo.go b/modules/repository/repo.go index d9dc82b03917..0067ca8de33a 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -7,6 +7,7 @@ package repository import ( "context" "fmt" + "io" "net/url" "path" "strings" @@ -323,60 +324,78 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi errChan := make(chan error, 1) go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) - err := func() error { - for pointerBlob := range pointerChan { - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID}) - if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) - } - if meta.Existing { - continue + downloadObjects := func(pointers []lfs.Pointer) error { + err := client.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { + if objectError != nil { + return objectError } - log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Oid, repo.FullName()) + defer content.Close() - err = func() error { - exist, err := contentStore.Exists(pointerBlob.Pointer) - if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err) - } - if !exist { - if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { - log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Oid, setting.LFS.MaxFileSize, pointerBlob.Size) - return nil - } - - stream, err := client.Download(ctx, pointerBlob.Pointer) - if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository: LFS OID[%s] failed to download: %w", pointerBlob.Oid, err) - } - defer stream.Close() - - if err := contentStore.Put(pointerBlob.Pointer, stream); err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository LFS OID[%s] contentStore.Put: %w", pointerBlob.Oid, err) - } - } else { - log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] already present in content store", pointerBlob.Oid) - } - return nil - }() + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repo.ID}) if err != nil { + return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) + } + + if err := contentStore.Put(p, content); err != nil { if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { log.Error("StoreMissingLfsObjectsInRepository RemoveLFSMetaObjectByOid[Oid: %s]: %w", meta.Oid, err2) } + return fmt.Errorf("StoreMissingLfsObjectsInRepository LFS OID[%s] contentStore.Put: %w", p.Oid, err) + } + return nil + }) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + } + return err + } - select { - case <-ctx.Done(): - return nil - default: + var batch []lfs.Pointer + for pointerBlob := range pointerChan { + meta, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid) + if err != nil && err != models.ErrLFSObjectNotExist { + return fmt.Errorf("StoreMissingLfsObjectsInRepository models.GetLFSMetaObjectByOid: %w", err) + } + if meta != nil { + continue + } + + log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Oid, repo.FullName()) + + exist, err := contentStore.Exists(pointerBlob.Pointer) + if err != nil { + return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err) + } + + if exist { + _, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID}) + if err != nil { + return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) + } + } else { + if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { + log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Oid, setting.LFS.MaxFileSize, pointerBlob.Size) + continue + } + + batch = append(batch, pointerBlob.Pointer) + if len(batch) >= client.BatchSize() { + if err := downloadObjects(batch); err != nil { + return err } - return err + batch = nil } } - return nil - }() - if err != nil { - return err + } + if len(batch) > 0 { + if err := downloadObjects(batch); err != nil { + return err + } } err, has := <-errChan diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index cb63c67b3a8e..55d810c0086f 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -7,7 +7,7 @@ package mirror import ( "context" "errors" - "fmt" + "io" "net/url" "strconv" "time" @@ -173,39 +173,48 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u errChan := make(chan error, 1) go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) - err := func() error { - for pointerBlob := range pointerChan { - err := func() error { - if exists, err := contentStore.Exists(pointerBlob.Pointer); !exists || err != nil { - return err - } - - content, err := contentStore.Get(pointerBlob.Pointer) - if err != nil { - return err - } - defer content.Close() - - if err := client.Upload(ctx, pointerBlob.Pointer, content); err != nil { - return fmt.Errorf("pushAllLFSObjects: LFS OID[%s] failed to upload: %w", pointerBlob.Oid, err) - } + uploadObjects := func(pointers []lfs.Pointer) error { + err := client.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { + if objectError != nil { + return nil, objectError + } + + return contentStore.Get(p) + }) + if err != nil { + select { + case <-ctx.Done(): return nil - }() - if err != nil { - select { - case <-ctx.Done(): - return nil - default: - } - return err + default: } } - return nil - }() - if err != nil { return err } + var batch []lfs.Pointer + for pointerBlob := range pointerChan { + exists, err := contentStore.Exists(pointerBlob.Pointer) + if err != nil { + return err + } + if !exists { + continue + } + + batch = append(batch, pointerBlob.Pointer) + if len(batch) >= client.BatchSize() { + if err := uploadObjects(batch); err != nil { + return err + } + batch = nil + } + } + if len(batch) > 0 { + if err := uploadObjects(batch); err != nil { + return err + } + } + err, has := <-errChan if has { return err From bacae7135c09be5259b7784f82b97168690221f6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Apr 2021 12:33:00 +0200 Subject: [PATCH 31/54] Added more logging. --- modules/lfs/http_client.go | 66 ++++++++++++++++++---------------- modules/lfs/transferadapter.go | 31 ++++++++-------- modules/repository/repo.go | 28 +++++++++------ services/mirror/mirror_push.go | 13 +++++-- 4 files changed, 79 insertions(+), 59 deletions(-) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 11fd4ed7f08b..af14ed9818da 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -62,6 +62,8 @@ func (c *HTTPClient) transferNames() []string { } func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { + log.Trace("BATCH operation with objects: %v", objects) + url := fmt.Sprintf("%s/objects/batch", c.endpoint) request := &BatchRequest{operation, c.transferNames(), nil, objects} @@ -69,14 +71,16 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin payload := new(bytes.Buffer) err := jsoniter.NewEncoder(payload).Encode(request) if err != nil { - return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err) + log.Error("Error encoding json: %v", err) + return nil, err } - log.Trace("lfs.HTTPClient.batch calling: %s", url) + log.Trace("Calling: %s", url) req, err := http.NewRequestWithContext(ctx, "POST", url, payload) if err != nil { - return nil, fmt.Errorf("lfs.HTTPClient.batch http.NewRequestWithContext: %w", err) + log.Error("Error creating request: %v", err) + return nil, err } req.Header.Set("Content-type", MediaType) req.Header.Set("Accept", MediaType) @@ -88,18 +92,20 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin return nil, ctx.Err() default: } - return nil, fmt.Errorf("lfs.HTTPClient.batch http.Do: %w", err) + log.Error("Error while processing request: %v", err) + return nil, err } defer res.Body.Close() if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("lfs.HTTPClient.batch: Unexpected servers response: %s", res.Status) + return nil, fmt.Errorf("Unexpected server response: %s", res.Status) } var response BatchResponse err = jsoniter.NewDecoder(res.Body).Decode(&response) if err != nil { - return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err) + log.Error("Error decoding json: %v", err) + return nil, err } if len(response.Transfer) == 0 { @@ -139,32 +145,40 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc return fmt.Errorf("TransferAdapter not found: %s", result.Transfer) } - if uc != nil { - for _, object := range result.Objects { - p := Pointer{object.Oid, object.Size} - - if object.Error != nil { - if _, err := uc(p, errors.New(object.Error.Message)); err != nil { + for _, object := range result.Objects { + if object.Error != nil { + objectError := errors.New(object.Error.Message) + log.Trace("Error on object %v: %v", object.Pointer, objectError) + if uc != nil { + if _, err := uc(object.Pointer, objectError); err != nil { + return err + } + } else { + if err := dc(object.Pointer, nil, objectError); err != nil { return err } - continue } + continue + } + if uc != nil { if len(object.Actions) == 0 { + log.Trace("%v already present on server", object.Pointer) continue } link, ok := object.Actions["upload"] if !ok { - return errors.New("Action 'upload' not found") + log.Debug("%+v", object) + return errors.New("Action 'upload' is missing") } - content, err := uc(p, nil) + content, err := uc(object.Pointer, nil) if err != nil { return err } - err = transferAdapter.Upload(ctx, link, p, content) + err = transferAdapter.Upload(ctx, link, object.Pointer, content) content.Close() @@ -174,25 +188,15 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc link, ok = object.Actions["verify"] if ok { - if err := transferAdapter.Verify(ctx, link, p); err != nil { + if err := transferAdapter.Verify(ctx, link, object.Pointer); err != nil { return err } } - } - } else { - for _, object := range result.Objects { - p := Pointer{object.Oid, object.Size} - - if object.Error != nil { - if err := dc(p, nil, errors.New(object.Error.Message)); err != nil { - return err - } - continue - } - + } else { link, ok := object.Actions["download"] if !ok { - return errors.New("Action 'download' not found") + log.Debug("%+v", object) + return errors.New("Action 'download' is mising") } content, err := transferAdapter.Download(ctx, link) @@ -200,7 +204,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc return err } - if err := dc(p, content, nil); err != nil { + if err := dc(object.Pointer, content, nil); err != nil { return err } } diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 9f86118321c9..8c40ab8c0446 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -46,7 +46,7 @@ func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCl // Upload sends the content to the LFS server func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { - _, err := a.performRequest(ctx, "PUT", l, r, func(req *http.Request) error { + _, err := a.performRequest(ctx, "PUT", l, r, func(req *http.Request) { if len(req.Header.Get("Content-Type")) == 0 { req.Header.Set("Content-Type", "application/octet-stream") } @@ -56,8 +56,6 @@ func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r } req.ContentLength = p.Size - - return nil }) if err != nil { return err @@ -69,13 +67,12 @@ func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error { b, err := jsoniter.Marshal(p) if err != nil { - return fmt.Errorf("lfs.BasicTransferAdapter.Verify json.Marshal: %w", err) + log.Error("Error encoding json: %v", err) + return err } - _, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b), func(req *http.Request) error { + _, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b), func(req *http.Request) { req.Header.Set("Content-Type", MediaType) - - return nil }) if err != nil { return err @@ -83,22 +80,21 @@ func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) e return nil } -func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, body io.Reader, rcb func(*http.Request) error) (*http.Response, error) { - log.Trace("lfs.BasicTransferAdapter.performRequest calling: %s %s", method, l.Href) +func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, body io.Reader, callback func(*http.Request)) (*http.Response, error) { + log.Trace("Calling: %s %s", method, l.Href) req, err := http.NewRequestWithContext(ctx, method, l.Href, body) if err != nil { - return nil, fmt.Errorf("lfs.BasicTransferAdapter.performRequest http.NewRequestWithContext: %w", err) + log.Error("Error creating request: %v", err) + return nil, err } for key, value := range l.Header { req.Header.Set(key, value) } req.Header.Set("Accept", MediaType) - if rcb != nil { - if err := rcb(req); err != nil { - return nil, err - } + if callback != nil { + callback(req) } res, err := a.client.Do(req) @@ -108,7 +104,8 @@ func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string return res, ctx.Err() default: } - return res, fmt.Errorf("lfs.BasicTransferAdapter.performRequest http.Do: %w", err) + log.Error("Error while processing request: %v", err) + return res, err } if res.StatusCode != http.StatusOK { @@ -125,11 +122,15 @@ func handleErrorResponse(resp *http.Response) error { if err != nil { return fmt.Errorf("Request failed with status %s", resp.Status) } + log.Trace("ErrorRespone: %v", er) return errors.New(er.Message) } func decodeReponseError(r io.Reader) (ErrorResponse, error) { var er ErrorResponse err := jsoniter.NewDecoder(r).Decode(&er) + if err != nil { + log.Error("Error decoding json: %v", err) + } return er, err } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 0067ca8de33a..08531c04ed3e 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -332,16 +332,18 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi defer content.Close() - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repo.ID}) + _, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repo.ID}) if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) + log.Error("Error creating LFS meta object %v: %v", p, err) + return err } if err := contentStore.Put(p, content); err != nil { - if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { - log.Error("StoreMissingLfsObjectsInRepository RemoveLFSMetaObjectByOid[Oid: %s]: %w", meta.Oid, err2) + log.Error("Error storing content for LFS meta object %v: %v", p, err) + if _, err2 := repo.RemoveLFSMetaObjectByOid(p.Oid); err2 != nil { + log.Error("Error removing LFS meta object %v: %v", p, err2) } - return fmt.Errorf("StoreMissingLfsObjectsInRepository LFS OID[%s] contentStore.Put: %w", p.Oid, err) + return err } return nil }) @@ -359,27 +361,32 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi for pointerBlob := range pointerChan { meta, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid) if err != nil && err != models.ErrLFSObjectNotExist { - return fmt.Errorf("StoreMissingLfsObjectsInRepository models.GetLFSMetaObjectByOid: %w", err) + log.Error("Error querying LFS meta object %v: %v", pointerBlob.Pointer, err) + return err } if meta != nil { + log.Trace("Skipping unknown LFS meta object %v", pointerBlob.Pointer) continue } - log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Oid, repo.FullName()) + log.Trace("LFS object %v not present in repository %s", pointerBlob.Pointer, repo.FullName()) exist, err := contentStore.Exists(pointerBlob.Pointer) if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err) + log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) + return err } if exist { + log.Trace("LFS object %v already present; creating meta object", pointerBlob.Pointer) _, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID}) if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) + log.Error("Error creating LFS meta object %v: %v", pointerBlob.Pointer, err) + return err } } else { if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { - log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Oid, setting.LFS.MaxFileSize, pointerBlob.Size) + log.Info("LFS object %v download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Pointer, setting.LFS.MaxFileSize, pointerBlob.Size) continue } @@ -400,6 +407,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi err, has := <-errChan if has { + log.Error("Error enumerating LFS objects for repository: %v", err) return err } diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 55d810c0086f..ac7ae9312637 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -112,8 +112,6 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second performPush := func(path string) error { - log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) - remoteAddr, err := git.GetRemoteAddress(path, m.RemoteName) if err != nil { log.Error("GetRemoteAddress(%s) Error %v", path, err) @@ -136,6 +134,8 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { } } + log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + if err := git.Push(path, git.PushOptions{ Remote: m.RemoteName, Force: true, @@ -179,7 +179,11 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u return nil, objectError } - return contentStore.Get(p) + content, err := contentStore.Get(p) + if err != nil { + log.Error("Error reading LFS object %v: %v", p, err) + } + return content, err }) if err != nil { select { @@ -195,9 +199,11 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u for pointerBlob := range pointerChan { exists, err := contentStore.Exists(pointerBlob.Pointer) if err != nil { + log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) return err } if !exists { + log.Trace("Skipping missing LFS object %v", pointerBlob.Pointer) continue } @@ -217,6 +223,7 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u err, has := <-errChan if has { + log.Error("Error enumerating LFS objects for repository: %v", err) return err } From 656c3478ba5265ef6b6ae1639c2735f1151788a8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 21 Apr 2021 17:55:33 +0000 Subject: [PATCH 32/54] Fixed unit tests. --- modules/lfs/http_client.go | 4 ++-- modules/lfs/http_client_test.go | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index af14ed9818da..e799b80831ea 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -170,7 +170,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc link, ok := object.Actions["upload"] if !ok { log.Debug("%+v", object) - return errors.New("Action 'upload' is missing") + return errors.New("Missing action 'upload'") } content, err := uc(object.Pointer, nil) @@ -196,7 +196,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc link, ok := object.Actions["download"] if !ok { log.Debug("%+v", object) - return errors.New("Action 'download' is mising") + return errors.New("Missing action 'download'") } content, err := transferAdapter.Download(ctx, link) diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index e2cc8f1649ee..98e693777a17 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -179,12 +179,12 @@ func TestHTTPClientDownload(t *testing.T) { // case 0 { endpoint: "https://status-not-ok.io", - expectederror: "Unexpected servers response: ", + expectederror: "Unexpected server response: ", }, // case 1 { endpoint: "https://invalid-json-response.io", - expectederror: "json.Decode: ", + expectederror: "invalid json", }, // case 2 { @@ -209,7 +209,7 @@ func TestHTTPClientDownload(t *testing.T) { // case 6 { endpoint: "https://empty-actions-map.io", - expectederror: "Action 'download' not found", + expectederror: "Missing action 'download'", }, // case 7 { @@ -219,17 +219,17 @@ func TestHTTPClientDownload(t *testing.T) { // case 8 { endpoint: "https://upload-actions-map.io", - expectederror: "Action 'download' not found", + expectederror: "Missing action 'download'", }, // case 9 { endpoint: "https://verify-actions-map.io", - expectederror: "Action 'download' not found", + expectederror: "Missing action 'download'", }, // case 10 { endpoint: "https://unknown-actions-map.io", - expectederror: "Action 'download' not found", + expectederror: "Missing action 'download'", }, } @@ -286,12 +286,12 @@ func TestHTTPClientUpload(t *testing.T) { // case 0 { endpoint: "https://status-not-ok.io", - expectederror: "Unexpected servers response: ", + expectederror: "Unexpected server response: ", }, // case 1 { endpoint: "https://invalid-json-response.io", - expectederror: "json.Decode: ", + expectederror: "invalid json", }, // case 2 { @@ -321,7 +321,7 @@ func TestHTTPClientUpload(t *testing.T) { // case 7 { endpoint: "https://download-actions-map.io", - expectederror: "Action 'upload' not found", + expectederror: "Missing action 'upload'", }, // case 8 { @@ -331,12 +331,12 @@ func TestHTTPClientUpload(t *testing.T) { // case 9 { endpoint: "https://verify-actions-map.io", - expectederror: "Action 'upload' not found", + expectederror: "Missing action 'upload'", }, // case 10 { endpoint: "https://unknown-actions-map.io", - expectederror: "Action 'upload' not found", + expectederror: "Missing action 'upload'", }, } From a6088cb1ca9f05ee0db0461dfaf3ac30da37b79f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 21 Apr 2021 18:01:41 +0000 Subject: [PATCH 33/54] Added created timestamp. --- models/migrations/v180.go | 5 +++-- models/repo_pushmirror.go | 6 ++++-- routers/repo/setting.go | 3 --- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/models/migrations/v180.go b/models/migrations/v180.go index 04cd97775b3d..7e451921f16b 100644 --- a/models/migrations/v180.go +++ b/models/migrations/v180.go @@ -20,8 +20,9 @@ func createPushMirrorTable(x *xorm.Engine) error { RemoteName string Interval time.Duration - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` - NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX next_update"` LastError string } diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index 383ebb960cd1..0727919624ff 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -27,14 +27,16 @@ type PushMirror struct { RemoteName string Interval time.Duration - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` - NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX next_update"` LastError string } // BeforeInsert will be invoked by XORM before inserting a record func (m *PushMirror) BeforeInsert() { if m != nil { + m.Created = timeutil.TimeStampNow() m.UpdatedUnix = 0 m.NextUpdateUnix = timeutil.TimeStampNow() } diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 075196b698bd..abaccc56550a 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -293,9 +293,6 @@ func SettingsPost(ctx *context.Context) { RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), Interval: interval, } - if interval != 0 { - m.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(interval) - } if err := models.InsertPushMirror(m); err != nil { ctx.ServerError("InsertPushMirror", err) return From 1923b514d8a5661f6ce3df5b7226fbefed6d8102 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 21 Apr 2021 20:17:24 +0000 Subject: [PATCH 34/54] Fixed name. --- models/repo_pushmirror.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index 0727919624ff..193641f1233b 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -36,7 +36,7 @@ type PushMirror struct { // BeforeInsert will be invoked by XORM before inserting a record func (m *PushMirror) BeforeInsert() { if m != nil { - m.Created = timeutil.TimeStampNow() + m.CreatedUnix = timeutil.TimeStampNow() m.UpdatedUnix = 0 m.NextUpdateUnix = timeutil.TimeStampNow() } From 5739008a3bd3a09e8752b2fe4eb4ca03dea7ef45 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 12 May 2021 16:43:00 +0200 Subject: [PATCH 35/54] Fixed invalid column name. --- models/repo_pushmirror.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index 193641f1233b..3292c5894184 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -110,8 +110,8 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { // PushMirrorsIterate iterates all push-mirror repositories. func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { return x. - Where("next_update_unix<=?", time.Now().Unix()). - And("next_update_unix!=0"). + Where("next_update<=?", time.Now().Unix()). + And("next_update!=0"). Iterate(new(PushMirror), f) } From 980548acf1ceffe4dbc102dac85d73cd2a82e8d7 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 12 May 2021 15:40:06 +0000 Subject: [PATCH 36/54] Changed name to prevent xorm auto setting. --- models/migrations/v180.go | 2 +- models/repo_pushmirror.go | 11 +---------- options/locale/locale_en-US.ini | 2 ++ routers/repo/setting.go | 9 +++++---- services/mirror/mirror_push.go | 2 +- templates/repo/settings/options.tmpl | 2 +- 6 files changed, 11 insertions(+), 17 deletions(-) diff --git a/models/migrations/v180.go b/models/migrations/v180.go index 7e451921f16b..0b1d2800e390 100644 --- a/models/migrations/v180.go +++ b/models/migrations/v180.go @@ -21,7 +21,7 @@ func createPushMirrorTable(x *xorm.Engine) error { Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX next_update"` LastError string } diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index 3292c5894184..b02fabf870e6 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -28,20 +28,11 @@ type PushMirror struct { Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX next_update"` LastError string } -// BeforeInsert will be invoked by XORM before inserting a record -func (m *PushMirror) BeforeInsert() { - if m != nil { - m.CreatedUnix = timeutil.TimeStampNow() - m.UpdatedUnix = 0 - m.NextUpdateUnix = timeutil.TimeStampNow() - } -} - // AfterLoad is invoked from XORM after setting the values of all fields of this object. func (m *PushMirror) AfterLoad(session *xorm.Session) { if m == nil { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5f770a3dff35..669590aa7e9b 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -94,6 +94,8 @@ step2 = Step 2: error = Error error404 = The page you are trying to reach either does not exist or you are not authorized to view it. +never = Never + [error] occurred = An error has occurred report_message = If you are sure this is a Gitea bug, please search for issue on GitHub and open new issue if necessary. diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 063ea900d555..827e52d24e47 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -288,10 +288,11 @@ func SettingsPost(ctx *context.Context) { } m := &models.PushMirror{ - RepoID: repo.ID, - Repo: repo, - RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), - Interval: interval, + RepoID: repo.ID, + Repo: repo, + RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), + Interval: interval, + NextUpdateUnix: timeutil.TimeStampNow(), } if err := models.InsertPushMirror(m); err != nil { ctx.ServerError("InsertPushMirror", err) diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index ac7ae9312637..1e6cfca35992 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -89,7 +89,7 @@ func syncPushMirror(ctx context.Context, mirrorID string) { return } - m.UpdatedUnix = timeutil.TimeStampNow() + m.LastUpdateUnix = timeutil.TimeStampNow() m.LastError = "" log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Running Sync", mirrorID, m.Repo) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 17e2b3024940..bf09a647ca2a 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -166,7 +166,7 @@ {{$address := MirrorRemoteAddress .}} {{$address.Address}} {{$.i18n.Tr "repo.settings.mirror_settings.direction.push"}} - {{.UpdatedUnix.AsTime}} {{if .LastError}}
{{$.i18n.Tr "error"}}
{{end}} + {{if .LastUpdateUnix}}{{.LastUpdateUnix.AsTime}}{{else}}{{$.i18n.Tr "never"}}{{end}} {{if .LastError}}
{{$.i18n.Tr "error"}}
{{end}}
{{$.CsrfTokenHtml}} From 310fdb98401af9769d1cbf68b0e09f62b71afac9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 12 May 2021 15:40:26 +0000 Subject: [PATCH 37/54] Remove table header im empty. --- templates/repo/settings/options.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index bf09a647ca2a..f6a6384d4ba1 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -76,6 +76,7 @@
+ {{if or .Repository.IsMirror .Repository.PushMirrors}} @@ -84,6 +85,7 @@ + {{end}} {{if .Repository.IsMirror}} @@ -184,7 +186,7 @@ {{else}} - + {{end}} From 33d0244a891f999114061c5d7c4246d0baa918ce Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 12 May 2021 15:45:47 +0000 Subject: [PATCH 38/54] Strip exit code from error message. --- services/mirror/mirror_push.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 1e6cfca35992..dee50432a7ed 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -9,6 +9,7 @@ import ( "errors" "io" "net/url" + "regexp" "strconv" "time" @@ -22,6 +23,8 @@ import ( "code.gitea.io/gitea/modules/util" ) +var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) + // AddPushMirrorRemote registers the push mirror remote. func AddPushMirrorRemote(m *models.PushMirror, addr string) error { addRemoteAndConfig := func(addr, path string) error { @@ -95,8 +98,8 @@ func syncPushMirror(ctx context.Context, mirrorID string) { log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Running Sync", mirrorID, m.Repo) err = runPushSync(ctx, m) if err != nil { - log.Error("SyncPushMirror [mirror: %s][repo: %-v]: %v", err) - m.LastError = err.Error() + log.Error("SyncPushMirror [mirror: %s][repo: %-v]: %v", mirrorID, m.Repo, err) + m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") } log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Scheduling next update", mirrorID, m.Repo) From ffaf55c5e411437fe536c9916830b46c6aa3733a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 13 May 2021 12:40:09 +0000 Subject: [PATCH 39/54] Added docs page about mirroring. --- .../content/doc/advanced/repo-mirror.en-us.md | 88 +++++++++++++++++++ options/locale/locale_en-US.ini | 1 + templates/repo/settings/options.tmpl | 1 + 3 files changed, 90 insertions(+) create mode 100644 docs/content/doc/advanced/repo-mirror.en-us.md diff --git a/docs/content/doc/advanced/repo-mirror.en-us.md b/docs/content/doc/advanced/repo-mirror.en-us.md new file mode 100644 index 000000000000..383c05f075ac --- /dev/null +++ b/docs/content/doc/advanced/repo-mirror.en-us.md @@ -0,0 +1,88 @@ +--- +date: "2019-09-06T01:35:00-03:00" +title: "Repository Mirror" +slug: "repo-mirror" +weight: 45 +toc: false +draft: false +menu: + sidebar: + parent: "advanced" + name: "Repository Mirror" + weight: 45 + identifier: "repo-mirror" +--- + +# Repository Mirror + +Repository mirroring allows for the mirroring of repositories to and from external sources. You can use it to mirror branches, tags, and commits between repositories. + +**Table of Contents** + +{{< toc >}} + +## Use cases + +The following are some possible use cases for repository mirroring: + +- You migrated to Gitea but still need to keep your project in another source. In that case, you can simply set it up to mirror to Gitea (pull) and all the essential history of commits, tags, and branches are available in your Gitea instance. +- You have old projects in another source that you don’t use actively anymore, but don’t want to remove for archiving purposes. In that case, you can create a push mirror so that your active Gitea repository can push its changes to the old location. + +## Pulling from a remote repository + +For an existing remote repository, you can set up pull mirroring as follows: + +1. Select **New Migration** in the **Create...** menu on the top right. +2. Select the remote repository service. +3. Enter a repository URL. +4. If the repository needs authentication fill in your authentication information. +5. Check the box **This repository will be a mirror**. +5. Select **Migrate repository** to save the configuration. + +The repository now gets mirrored periodically from the remote repository. You can force a sync by selecting **Synchronize Now** in the repository settings. + +## Pushing to a remote repository + +For an existing repository, you can set up push mirroring as follows: + +1. In your repository, go to **Settings** > **Repository**, and then the **Mirror Settings** section. +2. Enter a repository URL. +3. If the repository needs authentication expand the **Authorization** section and fill in your authentication information. +4. Select **Add Push Mirror** to save the configuration. + +The repository now gets mirrored periodically to the remote repository. You can force a sync by selecting **Synchronize Now**. In case of an error a message displayed to help you resolve it. + +:exclamation::exclamation: **NOTE:** This will force push to the remote repository. This will overwrite any changes in the remote repository! :exclamation::exclamation: + +### Setting up a push mirror from Gitea to GitHub + +To set up a mirror from Gitea to GitHub, you need to follow these steps: + +1. Create a [GitHub personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the *public_repo* box checked. +2. Fill in the **Git Remote Repository URL**: `https://github.com//.git`. +3. Fill in the **Authorization** fields with your GitHub username and the personal access token. +4. Select **Add Push Mirror** to save the configuration. + +The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button. + +### Setting up a push mirror from Gitea to GitLab + +To set up a mirror from Gitea to GitLab, you need to follow these steps: + +1. Create a [GitLab personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) with *write_repository* scope. +2. Fill in the **Git Remote Repository URL**: `https:////.git`. +3. Fill in the **Authorization** fields with `oauth2` as **Username** and your GitLab personal access token as **Password**. +4. Select **Add Push Mirror** to save the configuration. + +The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button. + +### Setting up a push mirror from Gitea to Bitbucket + +To set up a mirror from Gitea to Bitbucket, you need to follow these steps: + +1. Create a [Bitbucket app password](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/) with the *Repository Write* box checked. +2. Fill in the **Git Remote Repository URL**: `https://bitbucket.org//.git`. +3. Fill in the **Authorization** fields with your Bitbucket username and the app password as **Password**. +4. Select **Add Push Mirror** to save the configuration. + +The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button. diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 669590aa7e9b..7c1f968724e1 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1541,6 +1541,7 @@ settings.hooks = Webhooks settings.githooks = Git Hooks settings.basic_settings = Basic Settings settings.mirror_settings = Mirror Settings +settings.mirror_settings.docs = Set up your project to automatically push and/or pull changes to/from another repository. Branches, tags, and commits will be synced automatically. How do I mirror repositories? settings.mirror_settings.mirrored_repository = Mirrored repository settings.mirror_settings.direction = Direction settings.mirror_settings.direction.pull = Pull diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index f6a6384d4ba1..3681166cb3c4 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -75,6 +75,7 @@ {{.i18n.Tr "repo.settings.mirror_settings"}}
+ {{$.i18n.Tr "repo.settings.mirror_settings.docs" | Safe}}
{{$.i18n.Tr "repo.settings.mirror_settings.mirrored_repository"}}
{{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.none"}}{{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.none"}}
{{if or .Repository.IsMirror .Repository.PushMirrors}} From 180a61c0a8943fb7e6e3dcc8ccc8e1f0e9f2a8c0 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 18 May 2021 15:03:59 +0000 Subject: [PATCH 40/54] Fixed date. --- docs/content/doc/advanced/repo-mirror.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/advanced/repo-mirror.en-us.md b/docs/content/doc/advanced/repo-mirror.en-us.md index 383c05f075ac..bda5b0fa5594 100644 --- a/docs/content/doc/advanced/repo-mirror.en-us.md +++ b/docs/content/doc/advanced/repo-mirror.en-us.md @@ -1,5 +1,5 @@ --- -date: "2019-09-06T01:35:00-03:00" +date: "2021-05-13T00:00:00-00:00" title: "Repository Mirror" slug: "repo-mirror" weight: 45 From b83497bc7098f81288027e69adc63e1028662f2c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 18 May 2021 15:36:38 +0000 Subject: [PATCH 41/54] Use new method. --- routers/repo/setting.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 9ccc97373e9c..7197bdd71d80 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -17,7 +17,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/generate" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" @@ -26,6 +25,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/utils" @@ -281,9 +281,9 @@ func SettingsPost(ctx *context.Context) { return } - remoteSuffix, err := generate.GetRandomString(10) + remoteSuffix, err := util.RandomString(10) if err != nil { - ctx.ServerError("GetRandomString", err) + ctx.ServerError("RandomString", err) return } From 2a01584aace66e2b3291b335b7fdf77e00c176de Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 18 May 2021 15:36:50 +0000 Subject: [PATCH 42/54] Use jsoniter. --- modules/lfs/http_client_test.go | 8 ++++---- modules/lfs/transferadapter_test.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 98e693777a17..0f633ede54cd 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -7,13 +7,13 @@ package lfs import ( "bytes" "context" - "encoding/json" "io" "io/ioutil" "net/http" "strings" "testing" + jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" ) @@ -146,7 +146,7 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { } payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(batchResponse) + jsoniter.NewEncoder(payload).Encode(batchResponse) return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} } @@ -160,7 +160,7 @@ func TestHTTPClientDownload(t *testing.T) { assert.Equal(t, MediaType, req.Header.Get("Accept")) var batchRequest BatchRequest - err := json.NewDecoder(req.Body).Decode(&batchRequest) + err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest) assert.NoError(t, err) assert.Equal(t, "download", batchRequest.Operation) @@ -267,7 +267,7 @@ func TestHTTPClientUpload(t *testing.T) { assert.Equal(t, MediaType, req.Header.Get("Accept")) var batchRequest BatchRequest - err := json.NewDecoder(req.Body).Decode(&batchRequest) + err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest) assert.NoError(t, err) assert.Equal(t, "upload", batchRequest.Operation) diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index e409e095c067..7dfdad417ea5 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -7,13 +7,13 @@ package lfs import ( "bytes" "context" - "encoding/json" "io" "io/ioutil" "net/http" "strings" "testing" + jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" ) @@ -49,7 +49,7 @@ func TestBasicTransferAdapter(t *testing.T) { assert.Equal(t, MediaType, req.Header.Get("Content-Type")) var vp Pointer - err := json.NewDecoder(req.Body).Decode(&vp) + err := jsoniter.NewDecoder(req.Body).Decode(&vp) assert.NoError(t, err) assert.Equal(t, p.Oid, vp.Oid) assert.Equal(t, p.Size, vp.Size) @@ -60,7 +60,7 @@ func TestBasicTransferAdapter(t *testing.T) { Message: "Object not found", } payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(er) + jsoniter.NewEncoder(payload).Encode(er) return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(payload)} } else { From 902a7302a9603084e6cea8da6d39b9d60c3472ea Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 31 May 2021 19:04:00 +0200 Subject: [PATCH 43/54] Fixed merge errors. --- models/migrations/v180.go | 2 +- models/task.go | 2 +- modules/task/task.go | 2 +- routers/repo/setting.go | 7 +++---- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/models/migrations/v180.go b/models/migrations/v180.go index c2a3ff961a9e..a0471e151f9d 100644 --- a/models/migrations/v180.go +++ b/models/migrations/v180.go @@ -64,7 +64,7 @@ func removeCredentials(payload string) (string, error) { opts.AuthPassword = "" opts.AuthToken = "" - opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true) + opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr) confBytes, err := json.Marshal(opts) if err != nil { diff --git a/models/task.go b/models/task.go index a4ab65b5e5e1..2743d91f668f 100644 --- a/models/task.go +++ b/models/task.go @@ -234,7 +234,7 @@ func FinishMigrateTask(task *Task) error { } conf.AuthPassword = "" conf.AuthToken = "" - conf.CloneAddr = util.SanitizeURLCredentials(conf.CloneAddr, true) + conf.CloneAddr = util.NewStringURLSanitizer(conf.CloneAddr, true).Replace(conf.CloneAddr) conf.AuthPasswordEncrypted = "" conf.AuthTokenEncrypted = "" conf.CloneAddrEncrypted = "" diff --git a/modules/task/task.go b/modules/task/task.go index 0685aa23d743..1c0a87e1f61a 100644 --- a/modules/task/task.go +++ b/modules/task/task.go @@ -74,7 +74,7 @@ func CreateMigrateTask(doer, u *models.User, opts base.MigrateOptions) (*models. if err != nil { return nil, err } - opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true) + opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr) opts.AuthPasswordEncrypted, err = secret.EncryptSecret(setting.SecretKey, opts.AuthPassword) if err != nil { return nil, err diff --git a/routers/repo/setting.go b/routers/repo/setting.go index a90c3c2cf548..6690c5dbd4eb 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -170,10 +170,9 @@ func SettingsPost(ctx *context.Context) { } } - oldUsername := mirror_service.Username(ctx.Repo.Mirror) - oldPassword := mirror_service.Password(ctx.Repo.Mirror) - if form.MirrorPassword == "" && form.MirrorUsername == oldUsername { - form.MirrorPassword = oldPassword + u, _ := git.GetRemoteAddress(ctx.Repo.Repository.RepoPath(), ctx.Repo.Mirror.GetRemoteName()) + if u.User != nil && form.MirrorPassword == "" && form.MirrorUsername == u.User.Username() { + form.MirrorPassword, _ = u.User.Password() } address, err := forms.ParseRemoteAddr(form.MirrorAddress, form.MirrorUsername, form.MirrorPassword) From 68e0c00861f174249045467a0913d220746b97b4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 31 May 2021 19:08:00 +0200 Subject: [PATCH 44/54] Use syncPullMirror in test. --- services/mirror/mirror.go | 4 ++-- services/mirror/mirror_pull.go | 18 ++++++++++-------- .../{mirror_test.go => mirror_pull_test.go} | 5 +++-- services/mirror/mirror_push.go | 10 +++++++--- 4 files changed, 22 insertions(+), 15 deletions(-) rename services/mirror/{mirror_test.go => mirror_pull_test.go} (95%) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index aa1f52cd454b..8c2aa5dd1852 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -74,9 +74,9 @@ func syncMirrors(ctx context.Context) { return case item := <-mirrorQueue.Queue(): if strings.HasPrefix(item, "pull") { - syncPullMirror(ctx, item[5:]) + _ = syncPullMirror(ctx, item[5:]) } else if strings.HasPrefix(item, "push") { - syncPushMirror(ctx, item[5:]) + _ = syncPushMirror(ctx, item[5:]) } else { log.Error("Unknown item in queue: %v", item) } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 9b77d05c3431..c01e3eb501f3 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -255,7 +255,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) return parseRemoteUpdateOutput(output), true } -func syncPullMirror(ctx context.Context, repoID string) { +func syncPullMirror(ctx context.Context, repoID string) bool { log.Trace("SyncMirrors [repo_id: %v]", repoID) defer func() { err := recover() @@ -270,20 +270,20 @@ func syncPullMirror(ctx context.Context, repoID string) { m, err := models.GetMirrorByRepoID(id) if err != nil { log.Error("GetMirrorByRepoID [%s]: %v", repoID, err) - return + return false } log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) results, ok := runSync(ctx, m) if !ok { - return + return false } log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) m.ScheduleNextUpdate() if err = models.UpdateMirror(m); err != nil { log.Error("UpdateMirror [%s]: %v", repoID, err) - return + return false } var gitRepo *git.Repository @@ -294,12 +294,12 @@ func syncPullMirror(ctx context.Context, repoID string) { gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) if err != nil { log.Error("OpenRepository [%d]: %v", m.RepoID, err) - return + return false } defer gitRepo.Close() if ok := checkAndUpdateEmptyRepository(m, gitRepo, results); !ok { - return + return false } } @@ -374,15 +374,17 @@ func syncPullMirror(ctx context.Context, repoID string) { commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) if err != nil { log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err) - return + return false } if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil { log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) - return + return false } log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) + + return true } func checkAndUpdateEmptyRepository(m *models.Mirror, gitRepo *git.Repository, results []*mirrorSyncResult) bool { diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_pull_test.go similarity index 95% rename from services/mirror/mirror_test.go rename to services/mirror/mirror_pull_test.go index 20492c784bdb..2ab06e16d894 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_pull_test.go @@ -7,6 +7,7 @@ package mirror import ( "context" "path/filepath" + "strconv" "testing" "code.gitea.io/gitea/models" @@ -76,7 +77,7 @@ func TestRelease_MirrorDelete(t *testing.T) { err = mirror.GetMirror() assert.NoError(t, err) - _, ok := runSync(ctx, mirror.Mirror) + ok := syncPullMirror(ctx, strconv.FormatInt(mirror.ID, 10)) assert.True(t, ok) count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) @@ -87,7 +88,7 @@ func TestRelease_MirrorDelete(t *testing.T) { assert.NoError(t, err) assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true)) - _, ok = runSync(ctx, mirror.Mirror) + ok = syncPullMirror(ctx, strconv.FormatInt(mirror.ID, 10)) assert.True(t, ok) count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions) diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index dee50432a7ed..15e9c9c1f708 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -74,7 +74,7 @@ func RemovePushMirrorRemote(m *models.PushMirror) error { return nil } -func syncPushMirror(ctx context.Context, mirrorID string) { +func syncPushMirror(ctx context.Context, mirrorID string) bool { log.Trace("SyncPushMirror [mirror: %s]", mirrorID) defer func() { err := recover() @@ -89,7 +89,7 @@ func syncPushMirror(ctx context.Context, mirrorID string) { m, err := models.GetPushMirrorByID(id) if err != nil { log.Error("GetPushMirrorByID [%s]: %v", mirrorID, err) - return + return false } m.LastUpdateUnix = timeutil.TimeStampNow() @@ -104,11 +104,15 @@ func syncPushMirror(ctx context.Context, mirrorID string) { log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Scheduling next update", mirrorID, m.Repo) m.ScheduleNextUpdate() - if err = models.UpdatePushMirror(m); err != nil { + if err := models.UpdatePushMirror(m); err != nil { log.Error("UpdatePushMirror [%s]: %v", mirrorID, err) + + return false } log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Finished", mirrorID, m.Repo) + + return err == nil } func runPushSync(ctx context.Context, m *models.PushMirror) error { From ac879f8ad7f7f4608cad923971554d4332cbe670 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 31 May 2021 19:09:00 +0200 Subject: [PATCH 45/54] Fixed merge error. --- templates/repo/settings/options.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 23cb97b35aa5..163a1a66d2a8 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -123,19 +123,19 @@

{{.i18n.Tr "repo.mirror_address_desc"}}

-
+
{{.i18n.Tr "repo.need_auth"}}
- +
- +

{{.i18n.Tr "repo.mirror_password_help"}}

From 65f47daaaaf9812d5a984e6fc7efe3f68506f0db Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 31 May 2021 19:11:00 +0200 Subject: [PATCH 46/54] Moved test to integrations. --- .../mirror => integrations}/mirror_pull_test.go | 17 ++++++----------- services/mirror/mirror.go | 4 +++- services/mirror/mirror_pull.go | 14 ++++++-------- 3 files changed, 15 insertions(+), 20 deletions(-) rename {services/mirror => integrations}/mirror_pull_test.go (87%) diff --git a/services/mirror/mirror_pull_test.go b/integrations/mirror_pull_test.go similarity index 87% rename from services/mirror/mirror_pull_test.go rename to integrations/mirror_pull_test.go index 2ab06e16d894..0d595ec052cc 100644 --- a/services/mirror/mirror_pull_test.go +++ b/integrations/mirror_pull_test.go @@ -2,12 +2,10 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package mirror +package integrations import ( "context" - "path/filepath" - "strconv" "testing" "code.gitea.io/gitea/models" @@ -15,16 +13,13 @@ import ( migration "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/repository" release_service "code.gitea.io/gitea/services/release" + mirror_service "code.gitea.io/gitea/services/mirror" "github.com/stretchr/testify/assert" ) -func TestMain(m *testing.M) { - models.MainTest(m, filepath.Join("..", "..")) -} - -func TestRelease_MirrorDelete(t *testing.T) { - assert.NoError(t, models.PrepareTestDatabase()) +func TestMirrorPull(t *testing.T) { + defer prepareTestEnv(t)() user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) @@ -77,7 +72,7 @@ func TestRelease_MirrorDelete(t *testing.T) { err = mirror.GetMirror() assert.NoError(t, err) - ok := syncPullMirror(ctx, strconv.FormatInt(mirror.ID, 10)) + ok := mirror_service.SyncPullMirror(ctx, mirror.ID) assert.True(t, ok) count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) @@ -88,7 +83,7 @@ func TestRelease_MirrorDelete(t *testing.T) { assert.NoError(t, err) assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true)) - ok = syncPullMirror(ctx, strconv.FormatInt(mirror.ID, 10)) + ok = mirror_service.SyncPullMirror(ctx, mirror.ID) assert.True(t, ok) count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 8c2aa5dd1852..2fe669206cfd 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -7,6 +7,7 @@ package mirror import ( "context" "fmt" + "strconv" "strings" "code.gitea.io/gitea/models" @@ -74,7 +75,8 @@ func syncMirrors(ctx context.Context) { return case item := <-mirrorQueue.Queue(): if strings.HasPrefix(item, "pull") { - _ = syncPullMirror(ctx, item[5:]) + id, _ := strconv.ParseInt(item[5:], 10, 64) + _ = SyncPullMirror(ctx, id) } else if strings.HasPrefix(item, "push") { _ = syncPushMirror(ctx, item[5:]) } else { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index c01e3eb501f3..7fcb4c9266fa 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -7,7 +7,6 @@ package mirror import ( "context" "fmt" - "strconv" "strings" "time" @@ -255,7 +254,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) return parseRemoteUpdateOutput(output), true } -func syncPullMirror(ctx context.Context, repoID string) bool { +func SyncPullMirror(ctx context.Context, repoID int64) bool { log.Trace("SyncMirrors [repo_id: %v]", repoID) defer func() { err := recover() @@ -263,13 +262,12 @@ func syncPullMirror(ctx context.Context, repoID string) bool { return } // There was a panic whilst syncMirrors... - log.Error("PANIC whilst syncMirrors[%s] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) + log.Error("PANIC whilst syncMirrors[%d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) }() - id, _ := strconv.ParseInt(repoID, 10, 64) - m, err := models.GetMirrorByRepoID(id) + m, err := models.GetMirrorByRepoID(repoID) if err != nil { - log.Error("GetMirrorByRepoID [%s]: %v", repoID, err) + log.Error("GetMirrorByRepoID [%d]: %v", repoID, err) return false } @@ -282,7 +280,7 @@ func syncPullMirror(ctx context.Context, repoID string) bool { log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) m.ScheduleNextUpdate() if err = models.UpdateMirror(m); err != nil { - log.Error("UpdateMirror [%s]: %v", repoID, err) + log.Error("UpdateMirror [%d]: %v", m.RepoID, err) return false } @@ -320,7 +318,7 @@ func syncPullMirror(ctx context.Context, repoID string) bool { } commitID, err := gitRepo.GetRefCommitID(result.refName) if err != nil { - log.Error("gitRepo.GetRefCommitID [repo_id: %s, ref_name: %s]: %v", m.RepoID, result.refName, err) + log.Error("gitRepo.GetRefCommitID [repo_id: %d, ref_name: %s]: %v", m.RepoID, result.refName, err) continue } notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ From 2bc6102f03b723a132737a2ab56a27bfbca1b351 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Jun 2021 15:39:40 +0000 Subject: [PATCH 47/54] Added push mirror test. --- integrations/mirror_pull_test.go | 2 +- integrations/mirror_push_test.go | 86 ++++++++++++++++++++++++++++++++ services/mirror/mirror.go | 4 +- services/mirror/mirror_pull.go | 1 + services/mirror/mirror_push.go | 35 +++++++------ 5 files changed, 110 insertions(+), 18 deletions(-) create mode 100644 integrations/mirror_push_test.go diff --git a/integrations/mirror_pull_test.go b/integrations/mirror_pull_test.go index 0d595ec052cc..0e4da74fcf42 100644 --- a/integrations/mirror_pull_test.go +++ b/integrations/mirror_pull_test.go @@ -12,8 +12,8 @@ import ( "code.gitea.io/gitea/modules/git" migration "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/repository" - release_service "code.gitea.io/gitea/services/release" mirror_service "code.gitea.io/gitea/services/mirror" + release_service "code.gitea.io/gitea/services/release" "github.com/stretchr/testify/assert" ) diff --git a/integrations/mirror_push_test.go b/integrations/mirror_push_test.go new file mode 100644 index 000000000000..3191ef770444 --- /dev/null +++ b/integrations/mirror_push_test.go @@ -0,0 +1,86 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "context" + "fmt" + "net/http" + "net/url" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + mirror_service "code.gitea.io/gitea/services/mirror" + + "github.com/stretchr/testify/assert" +) + +func TestMirrorPush(t *testing.T) { + onGiteaRun(t, testMirrorPush) +} + +func testMirrorPush(t *testing.T, u *url.URL) { + defer prepareTestEnv(t)() + + setting.Migrations.AllowLocalNetworks = true + + user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) + srcRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) + + mirrorRepo, err := repository.CreateRepository(user, user, models.CreateRepoOptions{ + Name: "test-push-mirror", + }) + assert.NoError(t, err) + + ctx := NewAPITestContext(t, user.LowerName, srcRepo.Name) + + doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t) + + mirrors, err := models.GetPushMirrorsByRepoID(srcRepo.ID) + assert.NoError(t, err) + assert.Len(t, mirrors, 1) + + ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID) + assert.True(t, ok) + + srcGitRepo, err := git.OpenRepository(srcRepo.RepoPath()) + assert.NoError(t, err) + defer srcGitRepo.Close() + + srcCommit, err := srcGitRepo.GetBranchCommit("master") + assert.NoError(t, err) + + mirrorGitRepo, err := git.OpenRepository(mirrorRepo.RepoPath()) + assert.NoError(t, err) + defer mirrorGitRepo.Close() + + mirrorCommit, err := mirrorGitRepo.GetBranchCommit("master") + assert.NoError(t, err) + + assert.Equal(t, srcCommit.ID, mirrorCommit.ID) +} + +func doCreatePushMirror(ctx APITestContext, address, username, password string) func(t *testing.T) { + return func(t *testing.T) { + csrf := GetCSRF(t, ctx.Session, fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame))) + + req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)), map[string]string{ + "_csrf": csrf, + "action": "push-mirror-add", + "push_mirror_address": address, + "push_mirror_username": username, + "push_mirror_password": password, + "push_mirror_interval": "0", + }) + ctx.Session.MakeRequest(t, req, http.StatusFound) + + flashCookie := ctx.Session.GetCookie("macaron_flash") + assert.NotNil(t, flashCookie) + assert.Contains(t, flashCookie.Value, "success") + } +} diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 2fe669206cfd..368d301802a7 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -74,11 +74,11 @@ func syncMirrors(ctx context.Context) { mirrorQueue.Close() return case item := <-mirrorQueue.Queue(): + id, _ := strconv.ParseInt(item[5:], 10, 64) if strings.HasPrefix(item, "pull") { - id, _ := strconv.ParseInt(item[5:], 10, 64) _ = SyncPullMirror(ctx, id) } else if strings.HasPrefix(item, "push") { - _ = syncPushMirror(ctx, item[5:]) + _ = SyncPushMirror(ctx, id) } else { log.Error("Unknown item in queue: %v", item) } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 7fcb4c9266fa..a16724b36fef 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -254,6 +254,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) return parseRemoteUpdateOutput(output), true } +// SyncPullMirror starts the sync of the pull mirror and schedules the next run. func SyncPullMirror(ctx context.Context, repoID int64) bool { log.Trace("SyncMirrors [repo_id: %v]", repoID) defer func() { diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 15e9c9c1f708..1073cf75611c 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -10,7 +10,6 @@ import ( "io" "net/url" "regexp" - "strconv" "time" "code.gitea.io/gitea/models" @@ -74,43 +73,43 @@ func RemovePushMirrorRemote(m *models.PushMirror) error { return nil } -func syncPushMirror(ctx context.Context, mirrorID string) bool { - log.Trace("SyncPushMirror [mirror: %s]", mirrorID) +// SyncPushMirror starts the sync of the push mirror and schedules the next run. +func SyncPushMirror(ctx context.Context, mirrorID int64) bool { + log.Trace("SyncPushMirror [mirror: %d]", mirrorID) defer func() { err := recover() if err == nil { return } // There was a panic whilst syncPushMirror... - log.Error("PANIC whilst syncPushMirror[%s] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) + log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) }() - id, _ := strconv.ParseInt(mirrorID, 10, 64) - m, err := models.GetPushMirrorByID(id) + m, err := models.GetPushMirrorByID(mirrorID) if err != nil { - log.Error("GetPushMirrorByID [%s]: %v", mirrorID, err) + log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err) return false } m.LastUpdateUnix = timeutil.TimeStampNow() m.LastError = "" - log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Running Sync", mirrorID, m.Repo) + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo) err = runPushSync(ctx, m) if err != nil { - log.Error("SyncPushMirror [mirror: %s][repo: %-v]: %v", mirrorID, m.Repo, err) + log.Error("SyncPushMirror [mirror: %d][repo: %-v]: %v", m.ID, m.Repo, err) m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") } - log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Scheduling next update", mirrorID, m.Repo) + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Scheduling next update", m.ID, m.Repo) m.ScheduleNextUpdate() if err := models.UpdatePushMirror(m); err != nil { - log.Error("UpdatePushMirror [%s]: %v", mirrorID, err) + log.Error("UpdatePushMirror [%d]: %v", m.ID, err) return false } - log.Trace("SyncPushMirror [mirror: %s][repo: %-v]: Finished", mirrorID, m.Repo) + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Finished", m.ID, m.Repo) return err == nil } @@ -163,9 +162,15 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { } if m.Repo.HasWiki() { - err := performPush(m.Repo.WikiPath()) - if err != nil { - return err + wikiPath := m.Repo.WikiPath() + _, err := git.GetRemoteAddress(wikiPath, m.RemoteName) + if err == nil { + err := performPush(wikiPath) + if err != nil { + return err + } + } else { + log.Trace("Skipping wiki: No remote configured") } } From d7a3719f27c4359f323b987e6051da7ee95d97bb Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 2 Jun 2021 15:00:03 +0000 Subject: [PATCH 48/54] Removed NextUpdateUnix. --- models/repo_pushmirror.go | 14 ++------------ routers/repo/setting.go | 9 ++++----- services/mirror/mirror.go | 4 ++-- services/mirror/mirror_push.go | 5 ++--- 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index b02fabf870e6..50cc941e7d99 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -29,7 +29,6 @@ type PushMirror struct { Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` - NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX next_update"` LastError string } @@ -101,16 +100,7 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { // PushMirrorsIterate iterates all push-mirror repositories. func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { return x. - Where("next_update<=?", time.Now().Unix()). - And("next_update!=0"). + Where("last_update + (interval / ?) <= ?", time.Second, time.Now().Unix()). + And("interval != 0"). Iterate(new(PushMirror), f) } - -// ScheduleNextUpdate calculates and sets next update time. -func (m *PushMirror) ScheduleNextUpdate() { - if m.Interval != 0 { - m.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(m.Interval) - } else { - m.NextUpdateUnix = 0 - } -} diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 6690c5dbd4eb..fb0d55ad061b 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -293,11 +293,10 @@ func SettingsPost(ctx *context.Context) { } m := &models.PushMirror{ - RepoID: repo.ID, - Repo: repo, - RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), - Interval: interval, - NextUpdateUnix: timeutil.TimeStampNow(), + RepoID: repo.ID, + Repo: repo, + RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), + Interval: interval, } if err := models.InsertPushMirror(m); err != nil { ctx.ServerError("InsertPushMirror", err) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 368d301802a7..1e30c919e6d4 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -53,11 +53,11 @@ func Update(ctx context.Context) error { } if err := models.MirrorsIterate(handler); err != nil { - log.Trace("MirrorsIterate: %v", err) + log.Error("MirrorsIterate: %v", err) return err } if err := models.PushMirrorsIterate(handler); err != nil { - log.Trace("PushMirrorsIterate: %v", err) + log.Error("PushMirrorsIterate: %v", err) return err } log.Trace("Finished: Update") diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 1073cf75611c..de813036894b 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -91,7 +91,6 @@ func SyncPushMirror(ctx context.Context, mirrorID int64) bool { return false } - m.LastUpdateUnix = timeutil.TimeStampNow() m.LastError = "" log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo) @@ -101,8 +100,8 @@ func SyncPushMirror(ctx context.Context, mirrorID int64) bool { m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") } - log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Scheduling next update", m.ID, m.Repo) - m.ScheduleNextUpdate() + m.LastUpdateUnix = timeutil.TimeStampNow() + if err := models.UpdatePushMirror(m); err != nil { log.Error("UpdatePushMirror [%d]: %v", m.ID, err) From dad9adf588238bd3938a7a81e2321845a01d8aff Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 2 Jun 2021 15:02:28 +0000 Subject: [PATCH 49/54] and here --- models/migrations/v181.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/migrations/v181.go b/models/migrations/v181.go index 0b1d2800e390..5a999e24f78f 100644 --- a/models/migrations/v181.go +++ b/models/migrations/v181.go @@ -22,7 +22,6 @@ func createPushMirrorTable(x *xorm.Engine) error { Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` - NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX next_update"` LastError string } From 4e39abbc6353db81592e2d0bd8f49572a585a488 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 11 Jun 2021 14:17:31 +0200 Subject: [PATCH 50/54] Apply suggestions from code review --- models/repo_pushmirror.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index 50cc941e7d99..8b83e74b9dfa 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -69,7 +69,7 @@ func UpdatePushMirror(m *PushMirror) error { // DeletePushMirrorByID deletes a push-mirrors by ID func DeletePushMirrorByID(ID int64) error { - _, err := x.Delete(&PushMirror{ID: ID}) + _, err := x.ID(ID).Delete(&PushMirror{}) return err } @@ -81,8 +81,8 @@ func DeletePushMirrorsByRepoID(repoID int64) error { // GetPushMirrorByID returns push-mirror information. func GetPushMirrorByID(ID int64) (*PushMirror, error) { - m := &PushMirror{ID: ID} - has, err := x.Get(m) + m := &PushMirror{} + has, err := x.ID(ID).Get(m) if err != nil { return nil, err } else if !has { From 19d79112bda111487be1ebdd6a6f52998293352d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 11 Jun 2021 17:54:59 +0000 Subject: [PATCH 51/54] Fixed sql error. --- models/repo_pushmirror.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index 8b83e74b9dfa..87bf641108b6 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -100,7 +100,7 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { // PushMirrorsIterate iterates all push-mirror repositories. func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { return x. - Where("last_update + (interval / ?) <= ?", time.Second, time.Now().Unix()). - And("interval != 0"). + Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). + And("`interval` != 0"). Iterate(new(PushMirror), f) } From 444b96c3f0f65454f9cc4075628f2b384169966c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 11 Jun 2021 21:56:54 +0000 Subject: [PATCH 52/54] Added test. --- models/repo_pushmirror_test.go | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 models/repo_pushmirror_test.go diff --git a/models/repo_pushmirror_test.go b/models/repo_pushmirror_test.go new file mode 100644 index 000000000000..66c499b1c359 --- /dev/null +++ b/models/repo_pushmirror_test.go @@ -0,0 +1,49 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "testing" + "time" + + "code.gitea.io/gitea/modules/timeutil" + + "github.com/stretchr/testify/assert" +) + +func TestPushMirrorsIterate(t *testing.T) { + assert.NoError(t, PrepareTestDatabase()) + + now := timeutil.TimeStampNow() + + InsertPushMirror(&PushMirror{ + RemoteName: "test-1", + LastUpdateUnix: now, + Interval: 1, + }) + + long, _ := time.ParseDuration("24h") + InsertPushMirror(&PushMirror{ + RemoteName: "test-2", + LastUpdateUnix: now, + Interval: long, + }) + + InsertPushMirror(&PushMirror{ + RemoteName: "test-3", + LastUpdateUnix: now, + Interval: 0, + }) + + time.Sleep(1 * time.Millisecond) + + PushMirrorsIterate(func(idx int, bean interface{}) error { + m, ok := bean.(*PushMirror) + assert.True(t, ok) + assert.Equal(t, "test-1", m.RemoteName) + assert.Equal(t, m.RemoteName, m.GetRemoteName()) + return nil + }) +} From 567dcccb09b007f8d17a2249c6b4d147f1571d2b Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 14 Jun 2021 12:52:23 +0200 Subject: [PATCH 53/54] untouch models/migrations/v182.go --- models/migrations/v182.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/migrations/v182.go b/models/migrations/v182.go index eee03250f4ca..dd9a04f27e72 100644 --- a/models/migrations/v182.go +++ b/models/migrations/v182.go @@ -16,6 +16,7 @@ func addIssueResourceIndexTable(x *xorm.Engine) error { sess := x.NewSession() defer sess.Close() + if err := sess.Begin(); err != nil { return err } From 2c05692cc42ddb3ce79fa27f645bc7a4f67137e0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 14 Jun 2021 15:46:08 +0200 Subject: [PATCH 54/54] `xorm:"text"` --- models/migrations/v183.go | 2 +- models/repo_pushmirror.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/v183.go b/models/migrations/v183.go index 5a999e24f78f..cc752bf827c1 100644 --- a/models/migrations/v183.go +++ b/models/migrations/v183.go @@ -22,7 +22,7 @@ func createPushMirrorTable(x *xorm.Engine) error { Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` - LastError string + LastError string `xorm:"text"` } sess := x.NewSession() diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index 87bf641108b6..bdd4198f92b4 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -29,7 +29,7 @@ type PushMirror struct { Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` - LastError string + LastError string `xorm:"text"` } // AfterLoad is invoked from XORM after setting the values of all fields of this object.