From c67bef47cb1eba3dc4a3cba01666a920f7441931 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 12 Jul 2020 21:46:28 +0100 Subject: [PATCH 1/4] Remove double indirect in NewColoredIDValue Signed-off-by: Andrew Thornton --- modules/log/colors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/log/colors.go b/modules/log/colors.go index 282d0c919ef8e..d8e57767353a2 100644 --- a/modules/log/colors.go +++ b/modules/log/colors.go @@ -355,7 +355,7 @@ func NewColoredValueBytes(value interface{}, colorBytes *[]byte) *ColoredValue { // The Value will be colored with FgCyan // If a ColoredValue is provided it is not changed func NewColoredIDValue(value interface{}) *ColoredValue { - return NewColoredValueBytes(&value, &fgCyanBytes) + return NewColoredValueBytes(value, &fgCyanBytes) } // Format will format the provided value and protect against ANSI color spoofing within the value From 57d9a052f9c97fd476ff185e359b28e9aa986a42 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 13 Jul 2020 19:11:09 +0100 Subject: [PATCH 2/4] Handle forced-update in mirror.go Signed-off-by: Andrew Thornton --- services/mirror/mirror.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 7c12aa08589c5..4b3bd8e69a1da 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -157,6 +157,26 @@ func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { refName: refName, newCommitID: gitShortEmptySha, }) + case strings.HasPrefix(lines[i], " + "): // Force update + idx := strings.Index(refName, " ") + if idx > -1 { + refName = refName[:idx] + } + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error("SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "...") + if len(shas) != 2 { + log.Error("Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) case strings.HasPrefix(lines[i], " "): // New commits of a reference delimIdx := strings.Index(lines[i][3:], " ") if delimIdx == -1 { From 05950570555b6000bcf2fb0f488edd9fdba0fb70 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 13 Jul 2020 19:24:54 +0100 Subject: [PATCH 3/4] Add tracing Signed-off-by: Andrew Thornton --- services/mirror/mirror.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 4b3bd8e69a1da..867c6701c7174 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -207,6 +207,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { wikiPath := m.Repo.WikiPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) gitArgs := []string{"remote", "update"} if m.EnablePrune { gitArgs = append(gitArgs, "--prune") @@ -248,17 +249,21 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { log.Error("OpenRepository: %v", err) return nil, false } + + log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { gitRepo.Close() log.Error("Failed to synchronize tags to releases for repository: %v", err) } gitRepo.Close() + log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { log.Error("Failed to update size for mirror repository: %v", err) } if m.Repo.HasWiki() { + log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) stderrBuilder.Reset() stdoutBuilder.Reset() if err := git.NewCommand("remote", "update", "--prune"). @@ -288,8 +293,10 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { } return nil, false } + log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) } + log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) branches, err := repo_module.GetBranches(m.Repo) if err != nil { log.Error("GetBranches: %v", err) @@ -391,11 +398,13 @@ func syncMirror(repoID string) { } + log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) results, ok := runSync(m) if !ok { return } + log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) m.ScheduleNextUpdate() if err = models.UpdateMirror(m); err != nil { log.Error("UpdateMirror [%s]: %v", repoID, err) @@ -404,8 +413,9 @@ func syncMirror(repoID string) { var gitRepo *git.Repository if len(results) == 0 { - log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID) + log.Trace("SyncMirrors [repo: %-v]: no branches updated", m.Repo) } else { + log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) if err != nil { log.Error("OpenRepository [%d]: %v", m.RepoID, err) @@ -460,6 +470,7 @@ func syncMirror(repoID string) { notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, result.refName, oldCommitID, newCommitID, theCommits) } + log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) // Get latest commit date and update to current repository updated time commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) @@ -472,6 +483,8 @@ func syncMirror(repoID string) { log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) return } + + log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) } // InitSyncMirrors initializes a go routine to sync the mirrors From 8e27195c8d3f3b9c6f0677bfc9d8a8b0322cf639 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 22 Jul 2020 20:01:44 +0100 Subject: [PATCH 4/4] As per @lafriks Signed-off-by: Andrew Thornton --- services/mirror/mirror.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 867c6701c7174..75054b690f869 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -158,8 +158,7 @@ func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { newCommitID: gitShortEmptySha, }) case strings.HasPrefix(lines[i], " + "): // Force update - idx := strings.Index(refName, " ") - if idx > -1 { + if idx := strings.Index(refName, " "); idx > -1 { refName = refName[:idx] } delimIdx := strings.Index(lines[i][3:], " ")