Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Batch hook pre- and post-receive calls #8602

Merged
merged 24 commits into from
Dec 26, 2019
Merged

Batch hook pre- and post-receive calls #8602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
6bc317d
make notifyWatchers work on multiple actions
zeripath Oct 20, 2019
7434986
more efficient multiple notifyWatchers
zeripath Oct 20, 2019
2ed8271
Make CommitRepoAction take advantage of multiple actions
zeripath Nov 3, 2019
ca0a046
Batch post and pre-receive results
zeripath Oct 20, 2019
b2dbf87
Set batch to 30
zeripath Oct 21, 2019
ce4609b
Auto adjust timeout & add logging
zeripath Oct 22, 2019
fff0a1d
adjust processing message
zeripath Oct 23, 2019
66d8018
Add some messages to pre-receive
zeripath Oct 23, 2019
0114ae2
Make any non-200 status code from pre-receive an error
zeripath Nov 3, 2019
83ef52d
Add missing hookPrintResults
zeripath Nov 3, 2019
f177f16
Remove shortcut for single action
zeripath Nov 3, 2019
c0bb1f0
mistaken merge fix
zeripath Nov 3, 2019
52bc4ce
oops
zeripath Nov 11, 2019
494dd33
Merge branch 'master' into more-efficient-hooks
zeripath Dec 20, 2019
369674b
Merge branch 'master' into more-efficient-hooks
zeripath Dec 22, 2019
ad0f455
Move master branch to the front
zeripath Dec 22, 2019
dad23e7
If repo was empty and the master branch is pushed ensure that that is…
zeripath Dec 22, 2019
c633620
fixup
zeripath Dec 22, 2019
3d62415
fixup
zeripath Dec 22, 2019
1613c80
Missed HookOptions in setdefaultbranch
zeripath Dec 22, 2019
28d4886
Merge branch 'master' into more-efficient-hooks
lunny Dec 23, 2019
ac282e6
Batch PushUpdateAddTag and PushUpdateDelTag
zeripath Dec 25, 2019
de1ed4c
Merge branch 'master' into more-efficient-hooks
zeripath Dec 25, 2019
9d6f9f3
Merge branch 'master' into more-efficient-hooks
lunny Dec 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 174 additions & 43 deletions cmd/hook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ import (
"github.com/urfave/cli"
)

const (
hookBatchSize = 30
)

var (
// CmdHook represents the available hooks sub-command.
CmdHook = cli.Command{
Expand Down Expand Up @@ -75,12 +79,25 @@ Gitea or set your environment appropriately.`, "")
prID, _ := strconv.ParseInt(os.Getenv(models.ProtectedBranchPRID), 10, 64)
isDeployKey, _ := strconv.ParseBool(os.Getenv(models.EnvIsDeployKey))

buf := bytes.NewBuffer(nil)
hookOptions := private.HookOptions{
UserID: userID,
GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories),
GitObjectDirectory: os.Getenv(private.GitObjectDirectory),
GitQuarantinePath: os.Getenv(private.GitQuarantinePath),
ProtectedBranchID: prID,
IsDeployKey: isDeployKey,
}

scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
buf.Write(scanner.Bytes())
buf.WriteByte('\n')

oldCommitIDs := make([]string, hookBatchSize)
newCommitIDs := make([]string, hookBatchSize)
refFullNames := make([]string, hookBatchSize)
count := 0
total := 0
lastline := 0

for scanner.Scan() {
// TODO: support news feeds for wiki
if isWiki {
continue
Expand All @@ -94,29 +111,72 @@ Gitea or set your environment appropriately.`, "")
oldCommitID := string(fields[0])
newCommitID := string(fields[1])
refFullName := string(fields[2])
total++
lastline++

// If the ref is a branch, check if it's protected
if strings.HasPrefix(refFullName, git.BranchPrefix) {
statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{
OldCommitID: oldCommitID,
NewCommitID: newCommitID,
RefFullName: refFullName,
UserID: userID,
GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories),
GitObjectDirectory: os.Getenv(private.GitObjectDirectory),
GitQuarantinePath: os.Getenv(private.GitQuarantinePath),
ProtectedBranchID: prID,
IsDeployKey: isDeployKey,
})
switch statusCode {
case http.StatusInternalServerError:
fail("Internal Server Error", msg)
case http.StatusForbidden:
fail(msg, "")
oldCommitIDs[count] = oldCommitID
newCommitIDs[count] = newCommitID
refFullNames[count] = refFullName
count++
fmt.Fprintf(os.Stdout, "*")
os.Stdout.Sync()

if count >= hookBatchSize {
fmt.Fprintf(os.Stdout, " Checking %d branches\n", count)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was about to suggest exactly this to avoid the timeouts! 😄

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Anyway, this message will print after a line of stars and dots, is that what you intended?

****...*.*.***.****.*.***.****.***.*.*.*..*** Checking 30 branches

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup. that's intended

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if you count the preceding * in that line and the lines above there are 30 of them. The * indicates branches

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to disable these remote-messages?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please fire a new issue.

os.Stdout.Sync()

hookOptions.OldCommitIDs = oldCommitIDs
hookOptions.NewCommitIDs = newCommitIDs
hookOptions.RefFullNames = refFullNames
statusCode, msg := private.HookPreReceive(username, reponame, hookOptions)
switch statusCode {
case http.StatusOK:
// no-op
case http.StatusInternalServerError:
fail("Internal Server Error", msg)
default:
fail(msg, "")
}
count = 0
lastline = 0
}
} else {
fmt.Fprintf(os.Stdout, ".")
os.Stdout.Sync()
}
if lastline >= hookBatchSize {
fmt.Fprintf(os.Stdout, "\n")
os.Stdout.Sync()
lastline = 0
}
}

if count > 0 {
hookOptions.OldCommitIDs = oldCommitIDs[:count]
hookOptions.NewCommitIDs = newCommitIDs[:count]
hookOptions.RefFullNames = refFullNames[:count]

fmt.Fprintf(os.Stdout, " Checking %d branches\n", count)
os.Stdout.Sync()

statusCode, msg := private.HookPreReceive(username, reponame, hookOptions)
switch statusCode {
case http.StatusInternalServerError:
fail("Internal Server Error", msg)
case http.StatusForbidden:
fail(msg, "")
zeripath marked this conversation as resolved.
Show resolved Hide resolved
}
} else if lastline > 0 {
fmt.Fprintf(os.Stdout, "\n")
os.Stdout.Sync()
lastline = 0
}

fmt.Fprintf(os.Stdout, "Checked %d references in total\n", total)
os.Stdout.Sync()

return nil
}

Expand Down Expand Up @@ -156,12 +216,24 @@ Gitea or set your environment appropriately.`, "")
pusherID, _ := strconv.ParseInt(os.Getenv(models.EnvPusherID), 10, 64)
pusherName := os.Getenv(models.EnvPusherName)

buf := bytes.NewBuffer(nil)
hookOptions := private.HookOptions{
UserName: pusherName,
UserID: pusherID,
GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories),
GitObjectDirectory: os.Getenv(private.GitObjectDirectory),
GitQuarantinePath: os.Getenv(private.GitQuarantinePath),
}
oldCommitIDs := make([]string, hookBatchSize)
newCommitIDs := make([]string, hookBatchSize)
refFullNames := make([]string, hookBatchSize)
count := 0
total := 0
wasEmpty := false
masterPushed := false
results := make([]private.HookPostReceiveBranchResult, 0)

scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
buf.Write(scanner.Bytes())
buf.WriteByte('\n')

// TODO: support news feeds for wiki
if isWiki {
continue
Expand All @@ -172,36 +244,95 @@ Gitea or set your environment appropriately.`, "")
continue
}

oldCommitID := string(fields[0])
newCommitID := string(fields[1])
refFullName := string(fields[2])
fmt.Fprintf(os.Stdout, ".")
oldCommitIDs[count] = string(fields[0])
newCommitIDs[count] = string(fields[1])
refFullNames[count] = string(fields[2])
if refFullNames[count] == git.BranchPrefix+"master" && newCommitIDs[count] != git.EmptySHA && count == total {
masterPushed = true
}
count++
total++
os.Stdout.Sync()

if count >= hookBatchSize {
fmt.Fprintf(os.Stdout, " Processing %d references\n", count)
os.Stdout.Sync()
hookOptions.OldCommitIDs = oldCommitIDs
hookOptions.NewCommitIDs = newCommitIDs
hookOptions.RefFullNames = refFullNames
resp, err := private.HookPostReceive(repoUser, repoName, hookOptions)
if resp == nil {
hookPrintResults(results)
fail("Internal Server Error", err)
}
wasEmpty = wasEmpty || resp.RepoWasEmpty
results = append(results, resp.Results...)
count = 0
}
}

if count == 0 {
if wasEmpty && masterPushed {
// We need to tell the repo to reset the default branch to master
err := private.SetDefaultBranch(repoUser, repoName, "master")
if err != nil {
fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err)
}
}
fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total)
os.Stdout.Sync()

hookPrintResults(results)
return nil
zeripath marked this conversation as resolved.
Show resolved Hide resolved
}

res, err := private.HookPostReceive(repoUser, repoName, private.HookOptions{
OldCommitID: oldCommitID,
NewCommitID: newCommitID,
RefFullName: refFullName,
UserID: pusherID,
UserName: pusherName,
})
hookOptions.OldCommitIDs = oldCommitIDs[:count]
hookOptions.NewCommitIDs = newCommitIDs[:count]
hookOptions.RefFullNames = refFullNames[:count]

if res == nil {
fail("Internal Server Error", err)
fmt.Fprintf(os.Stdout, " Processing %d references\n", count)
os.Stdout.Sync()

resp, err := private.HookPostReceive(repoUser, repoName, hookOptions)
if resp == nil {
hookPrintResults(results)
fail("Internal Server Error", err)
}
wasEmpty = wasEmpty || resp.RepoWasEmpty
results = append(results, resp.Results...)

fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total)
os.Stdout.Sync()

if wasEmpty && masterPushed {
// We need to tell the repo to reset the default branch to master
err := private.SetDefaultBranch(repoUser, repoName, "master")
if err != nil {
fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err)
}
}

hookPrintResults(results)

if res["message"] == false {
return nil
}

func hookPrintResults(results []private.HookPostReceiveBranchResult) {
for _, res := range results {
if !res.Message {
continue
}

fmt.Fprintln(os.Stderr, "")
if res["create"] == true {
fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res["branch"])
fmt.Fprintf(os.Stderr, " %s\n", res["url"])
if res.Create {
fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res.Branch)
fmt.Fprintf(os.Stderr, " %s\n", res.URL)
} else {
fmt.Fprint(os.Stderr, "Visit the existing pull request:\n")
fmt.Fprintf(os.Stderr, " %s\n", res["url"])
fmt.Fprintf(os.Stderr, " %s\n", res.URL)
}
fmt.Fprintln(os.Stderr, "")
os.Stderr.Sync()
}

return nil
}
Loading