diff --git a/buildmodel/buildassets/buildassets.go b/buildmodel/buildassets/buildassets.go index 1ac39597..0839a0ae 100644 --- a/buildmodel/buildassets/buildassets.go +++ b/buildmodel/buildassets/buildassets.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io/ioutil" + "log" "os" "path" "sort" @@ -86,7 +87,7 @@ func (b BuildResultsDirectoryInfo) CreateSummary() (*BuildAssets, error) { if b.ArtifactsDir != "" { entries, err := os.ReadDir(b.ArtifactsDir) if err != nil { - panic(err) + log.Panic(err) } for _, e := range entries { @@ -154,7 +155,7 @@ func getVersion(path string, defaultVersion string) (version string) { if errors.Is(err, os.ErrNotExist) { return defaultVersion } - panic(err) + log.Panic(err) } return string(bytes) } @@ -162,7 +163,7 @@ func getVersion(path string, defaultVersion string) (version string) { func readFileOrPanic(path string) string { bytes, err := ioutil.ReadFile(path) if err != nil { - panic(err) + log.Panic(err) } return string(bytes) } diff --git a/buildmodel/commands.go b/buildmodel/commands.go index 0cdb955d..99a4b7d8 100644 --- a/buildmodel/commands.go +++ b/buildmodel/commands.go @@ -4,12 +4,13 @@ package buildmodel import ( - "errors" "flag" "fmt" + "log" "os" "os/exec" "path/filepath" + "time" "github.com/microsoft/go-infra/buildmodel/buildassets" "github.com/microsoft/go-infra/buildmodel/dockermanifest" @@ -136,8 +137,9 @@ func BindPRFlags() *PRFlags { // submits the resulting commit as a GitHub PR, approves with a second account, and enables the // GitHub auto-merge feature. func SubmitUpdatePR(f *PRFlags) error { - if _, err := os.Stat(*f.tempGitDir); !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("temporary Git dir already exists: %v", *f.tempGitDir) + gitDir, err := GetWorkPathInDir(*f.tempGitDir) + if err != nil { + return err } if *f.origin == "" { @@ -153,7 +155,7 @@ func SubmitUpdatePR(f *PRFlags) error { f.to = f.origin } - b := gitpr.PRBranch{ + b := gitpr.PRRefSet{ Name: *f.branch, Purpose: "auto-update", } @@ -168,6 +170,14 @@ func SubmitUpdatePR(f *PRFlags) error { return err } + title := fmt.Sprintf("Update dependencies in `%v`", b.Name) + body := fmt.Sprintf( + "🔃 This is an automatically generated PR updating the version of Go in `%v`.\n\n"+ + "This PR should auto-merge itself when PR validation passes.\n\n", + b.Name, + ) + request := b.CreateGitHubPR(parsedPRHeadRemote.GetOwner(), title, body) + // If we find a PR, fetch its head branch and push the new commit to its tip. We need to support // updating from many branches -> one branch, and force pushing each time would drop updates. // Note that we do assume our calculated head branch is the same as what the PR uses: it would @@ -181,10 +191,11 @@ func SubmitUpdatePR(f *PRFlags) error { if parsedOrigin != nil { fmt.Println("---- Checking for an existing PR for this base branch and origin...") existingPR, err = gitpr.FindExistingPR( - &b, + request, + parsedPRHeadRemote, + parsedOrigin, + b.PRBranch(), githubUser, - parsedPRHeadRemote.GetOwner(), - parsedOrigin.GetOwner(), *f.githubPAT) if err != nil { return err @@ -198,18 +209,18 @@ func SubmitUpdatePR(f *PRFlags) error { } // We're updating the target repo inside a clone of the go-infra repo, so we want a fresh clone. - runOrPanic(exec.Command("git", "init", *f.tempGitDir)) + runOrPanic(exec.Command("git", "init", gitDir)) // newGitCmd creates a "git {args}" command that runs in the temp git dir. newGitCmd := func(args ...string) *exec.Cmd { c := exec.Command("git", args...) - c.Dir = *f.tempGitDir + c.Dir = gitDir return c } if existingPR != "" { // Fetch the existing PR head branch to add onto. - runOrPanic(newGitCmd("fetch", "--no-tags", *f.to, b.PRBranchFetchRefspec())) + runOrPanic(newGitCmd("fetch", "--no-tags", *f.to, b.PRBranchRefspec())) } else { // Fetch the base branch to start the PR head branch. runOrPanic(newGitCmd("fetch", "--no-tags", *f.origin, b.BaseBranchFetchRefspec())) @@ -217,7 +228,7 @@ func SubmitUpdatePR(f *PRFlags) error { runOrPanic(newGitCmd("checkout", b.PRBranch())) // Make changes to the files ins the temp repo. - r, err := runUpdate(*f.tempGitDir, f) + r, err := runUpdate(gitDir, f) if err != nil { return err } @@ -229,7 +240,7 @@ func SubmitUpdatePR(f *PRFlags) error { fmt.Printf("---- Detected changes in Git stage. Continuing to commit and submit PR.\n") } else { // Make sure we don't ignore more than we intended. - panic(err) + log.Panic(err) } } else { // If the diff had 0 exit code, there are no changes. Skip this branch's next steps. @@ -240,7 +251,7 @@ func SubmitUpdatePR(f *PRFlags) error { runOrPanic(newGitCmd("commit", "-a", "-m", "Update "+b.Name+" to "+r.buildAssets.Version)) // Push the commit. - args := []string{"push", *f.origin, b.PRPushRefspec()} + args := []string{"push", *f.origin, b.PRBranchRefspec()} if *f.dryRun { // Show what would be pushed, but don't actually push it. args = append(args, "-n") @@ -277,7 +288,10 @@ func SubmitUpdatePR(f *PRFlags) error { if existingPR == "" { // POST the PR. The call returns success if the PR is created or if we receive a specific error // message back from GitHub saying the PR is already created. - p, err := gitpr.PostGitHub(parsedOrigin.GetOwnerSlashRepo(), b.CreateGitHubPR(parsedPRHeadRemote.GetOwner()), *f.githubPAT) + p, err := gitpr.PostGitHub( + parsedOrigin.GetOwnerSlashRepo(), + request, + *f.githubPAT) fmt.Printf("%+v\n", p) if err != nil { return err @@ -288,29 +302,13 @@ func SubmitUpdatePR(f *PRFlags) error { fmt.Printf("---- Submitted brand new PR: %v\n", p.HTMLURL) fmt.Printf("---- Approving with reviewer account...\n") - err = gitpr.MutateGraphQL( - *f.githubPATReviewer, - `mutation ($nodeID: ID!) { - addPullRequestReview(input: {pullRequestId: $nodeID, event: APPROVE, body: "Thanks! Auto-approving."}) { - clientMutationId - } - }`, - map[string]interface{}{"nodeID": p.NodeID}) - if err != nil { + if err = gitpr.ApprovePR(existingPR, *f.githubPATReviewer); err != nil { return err } } fmt.Printf("---- Enabling auto-merge with reviewer account...\n") - err = gitpr.MutateGraphQL( - *f.githubPATReviewer, - `mutation ($nodeID: ID!) { - enablePullRequestAutoMerge(input: {pullRequestId: $nodeID, mergeMethod: MERGE}) { - clientMutationId - } - }`, - map[string]interface{}{"nodeID": existingPR}) - if err != nil { + if err = gitpr.EnablePRAutoMerge(existingPR, *f.githubPATReviewer); err != nil { return err } @@ -319,6 +317,15 @@ func SubmitUpdatePR(f *PRFlags) error { return nil } +// GetWorkPathInDir creates a unique path inside the given root dir to use as a workspace. The name +// starts with the local time in a sortable format to help with browsing multiple workspaces. This +// function allows a command to run multiple times in sequence without overwriting or deleting the +// old data, for diagnostic purposes. +func GetWorkPathInDir(rootDir string) (string, error) { + pathDate := time.Now().Format("2006-01-02_15-04-05") + return os.MkdirTemp(rootDir, fmt.Sprintf("%s_*", pathDate)) +} + type updateResults struct { buildAssets *buildassets.BuildAssets } @@ -392,7 +399,7 @@ func runUpdate(repoRoot string, f *PRFlags) (*updateResults, error) { func getwd() string { wd, err := os.Getwd() if err != nil { - panic(err) + log.Panic(err) } return wd } @@ -400,7 +407,7 @@ func getwd() string { // runOrPanic uses 'run', then panics on error (such as nonzero exit code). func runOrPanic(c *exec.Cmd) { if err := run(c); err != nil { - panic(err) + log.Panic(err) } } diff --git a/cmd/dockerupdatepr/dockerupdatepr.go b/cmd/dockerupdatepr/dockerupdatepr.go index 610f41d0..563c55f8 100644 --- a/cmd/dockerupdatepr/dockerupdatepr.go +++ b/cmd/dockerupdatepr/dockerupdatepr.go @@ -5,6 +5,7 @@ package main import ( "fmt" + "log" "github.com/microsoft/go-infra/buildmodel" ) @@ -30,7 +31,7 @@ func main() { buildmodel.ParseBoundFlags(description) if err := buildmodel.SubmitUpdatePR(f); err != nil { - panic(err) + log.Panic(err) } fmt.Println("\nSuccess.") diff --git a/cmd/sync/model.go b/cmd/sync/model.go new file mode 100644 index 00000000..b1faa8be --- /dev/null +++ b/cmd/sync/model.go @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +// SyncConfigEntry is one entry in a sync config file. The file contains a JSON list of objects that +// match this struct. +type SyncConfigEntry struct { + // Upstream is the upstream Git repository to take updates from. + Upstream string + // Target is the GitHub repository to merge into, then submit the PR onto. It must be an https + // github.com URL. Other arguments passed to the sync tool may transform this URL into a + // different URL that works with authentication. + Target string + // Head is the GitHub repository to store the merged branch on. If not specified, defaults to + // the value of Target. This can be used to run the PR from a GitHub fork. + Head string + + // SourceBranches is the list of branches in Upstream to merge into Target. + SourceBranches []string + + // AutoResolveOurs contains files and dirs that upstream may modify, but we want to ignore those + // modifications and keep our changes to them. Normally our files are all in the 'eng/' + // directory, but some files are required by GitHub to be in the root of the repo or in the + // '.github' directory. In those cases, we must modify them in place and auto-resolve conflicts. + AutoResolveOurs []string +} diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 820ce375..4f796dae 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -4,216 +4,265 @@ package main import ( - "bytes" - "encoding/json" - "errors" "flag" "fmt" - "io/ioutil" - "net/http" + "log" "os" "os/exec" + "path" "path/filepath" + "strconv" "strings" - "time" + + "github.com/microsoft/go-infra/buildmodel" + "github.com/microsoft/go-infra/gitpr" ) const description = ` -Example: A sync operation dry run from upstream master to microsoft/go: - - eng/run.ps1 sync -b master -n +Example: A sync operation dry run: -It may be useful to specify Git addresses like 'git@github.com:microsoft/go' to -use SSH authentication. + go run ./cmd/sync -n -A 'sync' is a few steps to run "merge from upstream" and "mirror from upstream": +Sync runs a "merge from upstream" and submits it as a PR. This means fetching commits from an +upstream repo and merging them into corresponding branches in a target repo. This is configured in a +config file, by default 'eng/sync-config.json'. For each entry in the configuration: -1. Fetch every 'branch' from 'upstream'. -2. Fetch each 'microsoft/{branch}' from 'origin'. +1. Fetch each SourceBranch 'branch' from 'Upstream' to a local temp repository. +2. Fetch each 'microsoft/{branch}' from 'Target'. 3. Merge each upstream branch 'b' into corresponding 'microsoft/b'. -4. Push each merge commit to 'to' as 'auto-merge/microsoft/{branch}'. -5. Create a PR in 'origin' that merges the auto-merge branch. - - This PR is the "merge from upstream". -6. Push each branch from 'upstream' to 'to' with the exact same name. - - This push is the "mirror from upstream". - - We may change this to push to 'origin' in the future. See https://github.com/microsoft/go/issues/4 - -This script creates a temporary copy of the repository in 'eng/artifacts/' by -default. This avoids trampling changes in the user's clone.` - -// Files and dirs that upstream may modify, but we want to ignore those modifications and keep our -// changes to them. Normally our files are all in the 'eng/' directory, but some files are required -// by GitHub to be in the root of the repo or in the '.github' directory, so we must modify them in -// place and auto-resolve conflicts. -// -// This is in package scope just so it's easy to find at the top of the file for maintenance. -var autoResolveOurPaths = []string{ - ".gitattributes", - ".github", - "CODE_OF_CONDUCT.md", - "README.md", - "SECURITY.md", - "SUPPORT.md", -} +4. Push each merge commit to 'Head' (or 'Target' if 'Head' isn't specified) with a name that follows + the pattern 'dev/auto-merge/microsoft/{branch}'. +5. Create a PR in 'Target' that merges the auto-merge branch. If the PR already exists, overwrite. + (Force push.) + +This script creates the temporary repository in 'eng/artifacts/' by default. + +To run a subset of the syncs specified in the config file, or to swap out URLs for development +purposes, create a copy of the configuration file and point at it using a '-c' argument. +` var dryRun = flag.Bool("n", false, "Enable dry run: do not push, do not submit PR.") -var tempGitDir = flag.String("temp-git-dir", filepath.Join(getwd(), "eng", "artifacts", "sync-upstream-temp-repo"), "Location to create the temporary Git repo. Must not exist.") -var client = http.Client{ - Timeout: time.Second * 30, -} +var githubUser = flag.String("github-user", "", "Use this github user to submit pull requests.") +var githubPAT = flag.String("github-pat", "", "Submit the PR with this GitHub PAT, if specified.") +var githubPATReviewer = flag.String("github-pat-reviewer", "", "Approve the PR and turn on auto-merge with this PAT, if specified. Required, if github-pat specified.") func main() { - var to = flag.String("to", "https://github.com/microsoft-golang-bot/go", "Push synced refs to this Git repository.\n[Need push Git permission.]") - var origin = flag.String("origin", "https://github.com/microsoft/go", "Get latest 'microsoft/*' branches from this repo, and submit sync PR to this repo.\n[Need fetch Git permission.]") - var upstream = flag.String("upstream", "https://go.googlesource.com/go", "Get upstream Git data from this repo.\n[Need fetch Git permission.]") - - var githubPAT = flag.String("github-pat", "", "Submit the PR with this GitHub PAT, if specified.") - var githubPATReviewer = flag.String("github-pat-reviewer", "", "Approve the PR and turn on auto-merge with this PAT, if specified. Required, if github-pat specified.") - - var help = flag.Bool("h", false, "Print this help message.") - - var branchNames []string - flag.Func( - "b", - "Sync this upstream branch. Specify multiple times to sync multiple branches.\n"+ - "This must be the branch name as it's known by GitHub, like 'master'.", - func(arg string) error { - branchNames = append(branchNames, arg) - return nil - }) + var syncConfig = flag.String("c", "eng/sync-config.json", "The sync configuration file to run.") + var tempGitDir = flag.String( + "temp-git-dir", + filepath.Join(getwdOrPanic(), "eng", "artifacts", "sync-upstream-temp-repo"), + "Location to create the temporary Git repo. A timestamped subdirectory is created to reduce chance of collision.") + + var gitAuthSSH = flag.Bool("git-auth-ssh", false, "If enabled, automatically convert Target GitHub URLs into SSH format for authentication. 'git-auth-pat' is ignored if also specified.") + var gitAuthPAT = flag.Bool("git-auth-pat", false, "If enabled, automatically modify GitHub URLs to use 'github-user' and 'github-pat' for fetch/push access.") + + buildmodel.ParseBoundFlags(description) + + if *gitAuthPAT { + missingArgs := false + if *githubUser == "" { + fmt.Printf("Error: git-auth-pat is specified but github-user is not.") + missingArgs = true + } + if *githubPAT == "" { + fmt.Printf("Error: git-auth-pat is specified but github-pat is not.") + missingArgs = true + } + if missingArgs { + os.Exit(1) + } + } - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "\nUsage of sync-upstream-refs.go:\n") - flag.PrintDefaults() - fmt.Fprintf(flag.CommandLine.Output(), "%s\n\n", description) + var entries []SyncConfigEntry + if err := buildmodel.ReadJSONFile(*syncConfig, &entries); err != nil { + log.Panic(err) } - flag.Parse() + if len(entries) == 0 { + fmt.Printf("No entries found in config file: %v\n", *syncConfig) + } - if len(flag.Args()) > 0 { - fmt.Printf("Non-flag argument(s) provided but not accepted: %v\n", flag.Args()) - flag.Usage() - os.Exit(1) + currentRunGitDir, err := buildmodel.GetWorkPathInDir(*tempGitDir) + if err != nil { + log.Panic(err) } - if *help || len(branchNames) == 0 { - flag.Usage() - // Exit 0: script is successful, even though it didn't do anything except print usage info. - return + success := true + + for i, entry := range entries { + syncNum := fmt.Sprintf("%v/%v", i+1, len(entries)) + fmt.Printf("=== Beginning sync %v, from %v -> %v\n", syncNum, entry.Upstream, entry.Target) + + // Add authentication to Target URL if necessary. + targetRepoOwnerSlashName := strings.TrimPrefix(entry.Target, "https://github.com/") + if *gitAuthSSH { + entry.Target = fmt.Sprintf("git@github.com:%v", targetRepoOwnerSlashName) + } else if *gitAuthPAT { + entry.Target = fmt.Sprintf("https://%v:%v@github.com/%v", *githubUser, *githubPAT, targetRepoOwnerSlashName) + } + + if entry.Head == "" { + entry.Head = entry.Target + } + fmt.Printf("--- Head repository for PR: %v\n", entry.Head) + + // Give each entry a unique dir to avoid interfering with others upon failure. + repositoryDir := path.Join(currentRunGitDir, strconv.Itoa(i)) + + if err := syncRepository(repositoryDir, entry); err != nil { + // Let sync process continue if an error happens with the current entry. + fmt.Println(err) + fmt.Printf("=== Failed sync %v\n", syncNum) + success = false + } } - if _, err := os.Stat(*tempGitDir); !errors.Is(err, os.ErrNotExist) { - fmt.Printf("Error: Temporary Git dir already exists: %v\n", *tempGitDir) - os.Exit(1) + fmt.Println() + if success { + fmt.Println("Completed successfully.") + } else { + fmt.Println("Completed with errors.") } +} - runOrPanic(exec.Command("git", "init", *tempGitDir)) +// changedBranch stores the refs that have changes that need to be submitted in a PR, and the diff +// of files being changed in the PR for use in the PR body. +type changedBranch struct { + Refs *gitpr.SyncPRRefSet + Diff string +} - var branches []*branch = make([]*branch, 0, len(branchNames)) - for _, b := range branchNames { - nb := newBranch(b) - branches = append(branches, &nb) +func syncRepository(dir string, entry SyncConfigEntry) error { + if err := run(exec.Command("git", "init", dir)); err != nil { + return err } - // Fetch latest from remotes. We fetch with one big Git command per remote, instead of simply - // looping across every branch. This keeps round-trips to a minimum and benefits from innate Git - // parallelism. - // - // For each mirrored branch B in upstream, fetch it as 'auto-sync/B'. - // - // For each corresponding branch C in origin, fetch it as 'auto-merge/C'. (Branches in origin - // correspond to branches in upstream. E.g. 'master' B corresponds to 'microsoft/main' C.) - // - // Next, run auto-merge for each branch. We checkout 'auto-merge/C' and merge 'auto-sync/B' into - // it. This creates a merge commit in the local repo that brings origin 'microsoft/B' up to date - // with new changes in upstream 'B'. + // newGitCmd creates a "git {args}" command that runs in the temp fetch repo Git dir. + newGitCmd := func(args ...string) *exec.Cmd { + c := exec.Command("git", args...) + c.Dir = dir + return c + } + + branches := make([]*gitpr.SyncPRRefSet, 0, len(entry.SourceBranches)) + for _, b := range entry.SourceBranches { + nb := gitpr.NewSyncPRRefSet(b) + branches = append(branches, nb) + } + + // Fetch latest from remotes. We fetch with one big Git command with many refspecs, instead of + // simply looping across every branch. This keeps round-trips to a minimum and may benefit from + // innate Git parallelism. Later in the process, we do a batched "push" for the same reasons. // - // Once we're done merging each 'auto-merge/C' branch, we push all 'auto-sync/B' and - // 'auto-merge/C' branches to the 'to' repo. (Like fetching, this is also done with two commands - // to minimize round trips.) + // For an overview of the sequence of Git commands below, see the command description. - fetchUpstream := newGitCommand("fetch", "--no-tags", *upstream) - fetchOrigin := newGitCommand("fetch", "--no-tags", *origin) + fetchUpstream := newGitCmd("fetch", "--no-tags", entry.Upstream) + fetchOrigin := newGitCmd("fetch", "--no-tags", entry.Target) for _, b := range branches { - fetchUpstream.Args = append(fetchUpstream.Args, b.upstreamFetchRefspec()) - fetchOrigin.Args = append(fetchOrigin.Args, b.originFetchRefspec()) + fetchUpstream.Args = append(fetchUpstream.Args, b.UpstreamFetchRefspec()) + fetchOrigin.Args = append(fetchOrigin.Args, b.BaseBranchFetchRefspec()) + } + if err := run(fetchUpstream); err != nil { + return err + } + if err := run(fetchOrigin); err != nil { + return err } - runOrPanic(fetchUpstream) - runOrPanic(fetchOrigin) // While looping through the branches and trying to sync, use this slice to keep track of which // branches have changes, so we can push changes and submit PRs later. - changedBranches := make([]*branch, 0, len(branches)) + changedBranches := make([]changedBranch, 0, len(branches)) for _, b := range branches { - runOrPanic(newGitCommand("checkout", "auto-merge/"+b.mergeTarget)) + fmt.Printf("---- Processing branch '%v' for entry targeting %v\n", b.Name, entry.Target) - if err := run(newGitCommand("merge", "--no-ff", "--no-commit", "auto-sync/"+b.name)); err != nil { + if err := run(newGitCmd("checkout", b.PRBranch())); err != nil { + return err + } + + if err := run(newGitCmd("merge", "--no-ff", "--no-commit", b.UpstreamLocalBranch())); err != nil { if exitError, ok := err.(*exec.ExitError); ok { fmt.Printf("---- Merge hit an ExitError: '%v'. A non-zero exit code is expected if there were conflicts. The script will try to resolve them, next.\n", exitError) } else { // Make sure we don't ignore more than we intended. - panic(err) + return err } } - // Automatically resolve conflicts in specific project doc files. Use '--no-overlay' to make - // sure we delete new files in e.g. '.github' that are in upstream but don't exist locally. - // '--ours' auto-deletes if upstream modifies a file that we deleted in our branch. - runOrPanic(newGitCommand(append([]string{"checkout", "--no-overlay", "--ours", "HEAD", "--"}, autoResolveOurPaths...)...)) + if len(entry.AutoResolveOurs) > 0 { + // Automatically resolve conflicts in specific project doc files. Use '--no-overlay' to make + // sure we delete new files in e.g. '.github' that are in upstream but don't exist locally. + // '--ours' auto-deletes if upstream modifies a file that we deleted in our branch. + if err := run(newGitCmd(append([]string{"checkout", "--no-overlay", "--ours", "HEAD", "--"}, entry.AutoResolveOurs...)...)); err != nil { + return err + } + } // Check if there are any files in the stage. If not, we don't need to process this branch // anymore, because the merge + autoresolve didn't change anything. - if err := run(newGitCommand("diff", "--cached", "--quiet")); err != nil { + if err := run(newGitCmd("diff", "--cached", "--quiet")); err != nil { if _, ok := err.(*exec.ExitError); ok { fmt.Printf("---- Detected changes in Git stage. Continuing to commit and submit PR.\n") } else { // Make sure we don't ignore more than we intended. - panic(err) + return err } } else { // If the diff had 0 exit code, there are no changes. Skip this branch's next steps. - fmt.Printf("---- No changes to sync for %v. Skipping.\n", b.name) + fmt.Printf("---- No changes to sync for %v. Skipping.\n", b.Name) continue } // If we still have unmerged files, 'git commit' will exit non-zero, causing the script to // exit. This prevents the script from pushing a bad merge. - runOrPanic(newGitCommand("commit", "-m", "Merge upstream branch '"+b.name+"' into "+b.mergeTarget)) + if err := run(newGitCmd("commit", "-m", "Merge upstream branch '"+b.UpstreamName+"' into "+b.Name)); err != nil { + return err + } // Show a summary of which files are in our branch vs. upstream. This is just informational. // CI is a better place to *enforce* a low diff: it's more visible, can be fixed up more // easily, and doesn't block other branch mirror/merge operations. - // - // Save it to the branch struct so we can add it to the PR text. - b.fileDiff = combinedOutput(newGitCommand( + diff, err := combinedOutput(newGitCmd( "diff", "--name-status", - "auto-sync/"+b.name, - "auto-merge/"+b.mergeTarget, + b.UpstreamLocalBranch(), + b.PRBranch(), )) + if err != nil { + return err + } - fmt.Printf("---- Files changed from '%v' to '%v' ----\n", b.name, b.mergeTarget) - fmt.Print(b.fileDiff) + fmt.Printf("---- Files changed from '%v' to '%v' ----\n", b.UpstreamName, b.Name) + fmt.Print(diff) fmt.Println("--------") - changedBranches = append(changedBranches, b) + changedBranches = append(changedBranches, changedBranch{ + Refs: b, + Diff: diff, + }) } if len(changedBranches) == 0 { fmt.Println("Checked branches for changes to sync: none found.") fmt.Println("Success.") - return + return nil } - // Mirroring should always be FF: fail if not. This indicates upstream did some kind of a force - // push, so the merging probably wouldn't work anyway. - mirrorPushRefspecs := make([]string, 0, len(changedBranches)) - for _, b := range changedBranches { - mirrorPushRefspecs = append(mirrorPushRefspecs, b.mirrorPushRefspec()) + newGitPushCommand := func(remote string, force bool, refspecs []string) *exec.Cmd { + c := newGitCmd("push") + if force { + c.Args = append(c.Args, "--force") + } + c.Args = append(c.Args, remote) + for _, r := range refspecs { + c.Args = append(c.Args, r) + } + if *dryRun { + c.Args = append(c.Args, "-n") + } + return c } - runOrPanic(newGitPushCommand(*to, false, mirrorPushRefspecs)) // Force push the merge branches. We can't do a fast-forward push: our new merge commit is based // on "origin", not "to", so if "to" has any commits, they aren't in our commit's history. @@ -223,9 +272,11 @@ func main() { // simple and makes the PR flow simple. mergePushRefspecs := make([]string, 0, len(changedBranches)) for _, b := range changedBranches { - mergePushRefspecs = append(mergePushRefspecs, b.mergePushRefspec()) + mergePushRefspecs = append(mergePushRefspecs, b.Refs.PRBranchRefspec()) + } + if err := run(newGitPushCommand(entry.Head, true, mergePushRefspecs)); err != nil { + return err } - runOrPanic(newGitPushCommand(*to, true, mergePushRefspecs)) // All Git operations are complete! Next, ensure there's a GitHub PR for each auto-merge branch. @@ -233,10 +284,15 @@ func main() { // specific branch. var prFailed bool - // Lazy var. github user that owns the PRs. This is normally the owner of the 'to' repo. - var githubUser string - // Lazy var. The origin that should receive the PR. - var parsedOrigin *remote + // Parse the URLs involved in the PR to get segment information. + parsedPRTargetRemote, err := gitpr.ParseRemoteURL(entry.Target) + if err != nil { + return err + } + parsedPRHeadRemote, err := gitpr.ParseRemoteURL(entry.Head) + if err != nil { + return err + } for _, b := range changedBranches { var skipReason string @@ -244,6 +300,8 @@ func main() { case *dryRun: skipReason = "Dry run" + case *githubUser == "": + skipReason = "github-user not provided" case *githubPAT == "": skipReason = "github-pat not provided" @@ -254,24 +312,13 @@ func main() { skipReason = "github-pat-reviewer not provided" } + prFlowDescription := fmt.Sprintf("%v -> %v", b.Refs.UpstreamName, b.Refs.PRBranch()) + if skipReason != "" { - fmt.Printf("---- %s: skipping submitting PR for %v -> %v\n", skipReason, b.name, b.mergeTarget) + fmt.Printf("---- %s: skipping submitting PR for %v\n", skipReason, prFlowDescription) continue } - if githubUser == "" { - githubUser = getUsername(*githubPAT) - fmt.Printf("---- User for github-pat is: %v\n", githubUser) - } - - if parsedOrigin == nil { - var err error - if parsedOrigin, err = parseRemoteURL(*origin); err != nil { - fmt.Println(err) - os.Exit(1) - } - } - // err contains any err we get from running the sequence of GitHub PR submission API calls. // // This uses an immediately invoked anonymous function for convenience/maintainability. We @@ -279,52 +326,59 @@ func main() { // capture vars from the 'main()' scope rather than making them global or explicitly passing // each one into a named function. err := func() error { - fmt.Printf("---- PR for %v -> %v: Submitting...\n", b.name, b.mergeTarget) + fmt.Printf("---- PR for %v: Submitting...\n", prFlowDescription) + + title := fmt.Sprintf("Merge upstream `%v` into `%v`", b.Refs.UpstreamName, b.Refs.Name) + body := fmt.Sprintf( + "🔃 This is an automatically generated PR merging upstream `%v` into `%v`.\n\n"+ + "This PR should auto-merge itself when PR validation passes. If CI fails and you need to make fixups, be sure to use a merge commit, not a squash or rebase!\n\n"+ + "---\n\n"+ + "After these changes, the difference between upstream and the branch is:\n\n"+ + "```\n%v\n```", + b.Refs.UpstreamName, + b.Refs.Name, + strings.TrimSpace(b.Diff), + ) + request := b.Refs.CreateGitHubPR(parsedPRHeadRemote.GetOwner(), title, body) // POST the PR. The call returns success if the PR is created or if we receive a // specific error message back from GitHub saying the PR is already created. - pr, err := postPR(parsedOrigin.getOwnerSlashRepo(), b.createPRRequest(githubUser), *githubPAT) + pr, err := gitpr.PostGitHub(parsedPRTargetRemote.GetOwnerSlashRepo(), request, *githubPAT) fmt.Printf("%+v\n", pr) - if err != nil { return err } if pr.AlreadyExists { fmt.Println("---- A PR already exists. Attempting to find it...") - pr.NodeID, err = findExistingPR(b, githubUser, parsedOrigin.getOwner(), *githubPAT) + pr.NodeID, err = gitpr.FindExistingPR( + request, + parsedPRHeadRemote, + parsedPRTargetRemote, + b.Refs.PRBranch(), + *githubUser, + *githubPAT) if err != nil { return err } + if pr.NodeID == "" { + return fmt.Errorf("no PR found") + } } else { fmt.Printf("---- Submitted brand new PR: %v\n", pr.HTMLURL) fmt.Printf("---- Approving with reviewer account...\n") - err = mutateGraphQL( - *githubPATReviewer, - `mutation { - addPullRequestReview(input: {pullRequestId: "`+pr.NodeID+`", event: APPROVE, body: "Thanks! Auto-approving."}) { - clientMutationId - } - }`) - if err != nil { + if err = gitpr.ApprovePR(pr.NodeID, *githubPATReviewer); err != nil { return err } } fmt.Printf("---- Enabling auto-merge with reviewer account...\n") - err = mutateGraphQL( - *githubPATReviewer, - `mutation { - enablePullRequestAutoMerge(input: {pullRequestId: "`+pr.NodeID+`", mergeMethod: MERGE}) { - clientMutationId - } - }`) - if err != nil { + if err = gitpr.EnablePRAutoMerge(pr.NodeID, *githubPATReviewer); err != nil { return err } - fmt.Printf("---- PR for %v -> %v: Done.\n", b.name, b.mergeTarget) + fmt.Printf("---- PR for %v: Done.\n", prFlowDescription) return nil }() @@ -346,29 +400,21 @@ func main() { // If PR submission failed for any branch, exit the overall script with NZEC. if prFailed { - fmt.Printf("Failed to submit one or more PRs.") - os.Exit(1) + return fmt.Errorf("failed to submit one or more PRs") } - fmt.Println("\nSuccess.") + return nil } -// getwd gets the current working dir or panics, for easy use in expressions. -func getwd() string { +// getwdOrPanic gets the current working dir or panics, for easy use in expressions. +func getwdOrPanic() string { wd, err := os.Getwd() if err != nil { - panic(err) + log.Panic(err) } return wd } -// runOrPanic uses 'run', then panics on error (such as nonzero exit code). -func runOrPanic(c *exec.Cmd) { - if err := run(c); err != nil { - panic(err) - } -} - // run sets up the command so it logs directly to our stdout/stderr streams, then runs it. func run(c *exec.Cmd) error { fmt.Printf("---- Running command: %v %v\n", c.Path, c.Args) @@ -377,373 +423,12 @@ func run(c *exec.Cmd) error { return c.Run() } -// combinedOutput returns the output string of c.CombinedOutput, and panics on error. -func combinedOutput(c *exec.Cmd) string { +// combinedOutput returns the output string of c.CombinedOutput. +func combinedOutput(c *exec.Cmd) (string, error) { fmt.Printf("---- Running command: %v %v\n", c.Path, c.Args) out, err := c.CombinedOutput() if err != nil { - panic(err) - } - return string(out) -} - -func newGitCommand(args ...string) *exec.Cmd { - c := exec.Command("git", args...) - c.Dir = *tempGitDir - return c -} - -func newGitPushCommand(remote string, force bool, refspecs []string) *exec.Cmd { - c := newGitCommand("push") - if force { - c.Args = append(c.Args, "--force") - } - c.Args = append(c.Args, remote) - for _, r := range refspecs { - c.Args = append(c.Args, r) - } - if *dryRun { - c.Args = append(c.Args, "-n") - } - return c -} - -// branch contains information about a specific branch to sync. During the sync process, more info -// can be added to this struct to be used later. This struct has methods that help calculate derived -// information such as Git ref names. -type branch struct { - name string - mergeTarget string - - // fileDiff starts empty. It's filled in after the sync performs the merge. It contains a - // file-level diff between the upstream branch and the merge target. - fileDiff string -} - -func newBranch(b string) branch { - return branch{ - name: b, - mergeTarget: "microsoft/" + strings.ReplaceAll(b, "master", "main"), - } -} - -func (b branch) upstreamFetchRefspec() string { - return "refs/heads/" + b.name + ":refs/heads/auto-sync/" + b.name -} - -func (b branch) mirrorPushRefspec() string { - return "auto-sync/" + b.name + ":refs/heads/" + b.name -} - -func (b branch) originFetchRefspec() string { - return "refs/heads/" + b.mergeTarget + ":refs/heads/auto-merge/" + b.mergeTarget -} - -func (b branch) mergePushRefspec() string { - return "auto-merge/" + b.mergeTarget + ":refs/heads/auto-merge/" + b.mergeTarget -} - -func (b branch) createPRRequest(githubUser string) prRequest { - return prRequest{ - Head: githubUser + ":auto-merge/" + b.mergeTarget, - Base: b.mergeTarget, - - Title: fmt.Sprintf("Merge upstream `%v` into `%v`", b.name, b.mergeTarget), - Body: fmt.Sprintf( - "🔃 This is an automatically generated PR merging upstream `%v` into `%v`.\n\n"+ - "This PR should auto-merge itself when PR validation passes. If CI fails and you need to make fixups, be sure to use a merge commit, not a squash or rebase!\n\n"+ - "---\n\n"+ - "After these changes, the difference between upstream and the branch is:\n\n"+ - "```\n%v\n```", - b.name, - b.mergeTarget, - strings.TrimSpace(b.fileDiff), - ), - - MaintainerCanModify: true, - Draft: false, - } -} - -// remote is a parsed version of a Git remote. It helps determine how to send a GitHub PR. -type remote struct { - url string - urlParts []string -} - -// parseRemoteURL takes the URL ("https://github.com/microsoft/go", "git@github.com:microsoft/go") -// and grabs the owner ("microsoft") and repository name ("go"). This assumes the URL follows one of -// these two patterns, or something that's compatible. Returns an initialized 'remote'. -func parseRemoteURL(url string) (*remote, error) { - r := &remote{ - url, - strings.FieldsFunc(url, func(r rune) bool { return r == '/' || r == ':' }), - } - if len(r.urlParts) < 3 { - return r, fmt.Errorf( - "failed to find 3 parts of remote url '%v'. Found '%v'. Expected a string separated with '/' or ':', like https://github.com/microsoft/go or git@github.com:microsoft/go", - r.url, - r.urlParts, - ) - } - fmt.Printf("From repo URL %v, detected %v for the PR target.\n", url, r.urlParts) - return r, nil -} - -func (r remote) getOwnerRepo() []string { - return r.urlParts[len(r.urlParts)-2:] -} - -func (r remote) getOwner() string { - return r.getOwnerRepo()[0] -} - -func (r remote) getOwnerSlashRepo() string { - return strings.Join(r.getOwnerRepo(), "/") -} - -// sendJSONRequest sends a request for JSON information. The JSON response is unmarshalled (parsed) -// into the 'response' parameter, based on the structure of 'response'. -func sendJSONRequest(request *http.Request, response interface{}) (status int, err error) { - request.Header.Add("Accept", "application/vnd.github.v3+json") - fmt.Printf("Sending request: %v %v\n", request.Method, request.URL) - - httpResponse, err := client.Do(request) - if err != nil { - return - } - defer httpResponse.Body.Close() - status = httpResponse.StatusCode - - for key, value := range httpResponse.Header { - if strings.HasPrefix(key, "X-Ratelimit-") { - fmt.Printf("%v : %v\n", key, value) - } - } - - jsonBytes, err := ioutil.ReadAll(httpResponse.Body) - if err != nil { - return - } - - fmt.Printf("---- Full response:\n%v\n", string(jsonBytes)) - fmt.Printf("----\n") - - err = json.Unmarshal(jsonBytes, response) - return -} - -// sendJSONRequestSuccessful sends a request for JSON information via sendJsonRequest and verifies -// the status code is success. -func sendJSONRequestSuccessful(request *http.Request, response interface{}) error { - status, err := sendJSONRequest(request, response) - if err != nil { - return err - } - if status < 200 || status > 299 { - return fmt.Errorf("request unsuccessful, http status %v, %v", status, http.StatusText(status)) - } - return nil -} - -// getUsername queries GitHub for the username associated with a PAT. -func getUsername(pat string) string { - request, err := http.NewRequest("GET", "https://api.github.com/user", nil) - if err != nil { - panic(err) - } - request.SetBasicAuth("", pat) - - response := &struct { - Login string `json:"login"` - }{} - - if err := sendJSONRequestSuccessful(request, response); err != nil { - panic(err) - } - - return response.Login -} - -// prRequest is the payload for a GitHub PR creation API call, marshallable as JSON. -type prRequest struct { - Head string `json:"head"` - Base string `json:"base"` - Title string `json:"title"` - Body string `json:"body"` - MaintainerCanModify bool `json:"maintainer_can_modify"` - Draft bool `json:"draft"` -} - -// prRequestResponse is a PR creation response from GitHub. It may represent success or failure. -type prRequestResponse struct { - // GitHub success response: - HTMLURL string `json:"html_url"` - NodeID string `json:"node_id"` - - // GitHub failure response: - Message string `json:"message"` - Errors []prRequestError `json:"errors"` - - // AlreadyExists is set to true if the error message says the PR exists. Otherwise, false. For - // our purposes, a GitHub failure response that indicates a PR already exists is not an error. - AlreadyExists bool -} - -type prRequestError struct { - Message string `json:"message"` -} - -func postPR(ownerRepo string, request prRequest, pat string) (response *prRequestResponse, err error) { - prSubmitContent, err := json.MarshalIndent(request, "", "") - fmt.Printf("Submitting payload: %s\n", prSubmitContent) - - httpRequest, err := http.NewRequest("POST", "https://api.github.com/repos/"+ownerRepo+"/pulls", bytes.NewReader(prSubmitContent)) - if err != nil { - return - } - httpRequest.SetBasicAuth("", pat) - - response = &prRequestResponse{} - statusCode, err := sendJSONRequest(httpRequest, response) - if err != nil { - return - } - - switch statusCode { - case http.StatusCreated: - // 201 Created is the expected code if the PR is created. Do nothing. - - case http.StatusUnprocessableEntity: - // 422 Unprocessable Entity may indicate the PR already exists. GitHub also gives us a response - // that looks like this: - /* - { - "message": "Validation Failed", - "errors": [ - { - "resource": "PullRequest", - "code": "custom", - "message": "A pull request already exists for microsoft-golang-bot:auto-merge/microsoft/main." - } - ], - "documentation_url": "https://docs.github.com/rest/reference/pulls#create-a-pull-request" - } - */ - for _, e := range response.Errors { - if strings.HasPrefix(e.Message, "A pull request already exists for ") { - response.AlreadyExists = true - } - } - if !response.AlreadyExists { - err = fmt.Errorf( - "response code %v may indicate PR already exists, but the error message is not recognized: %v", - statusCode, - response.Errors, - ) - } - - default: - err = fmt.Errorf("unexpected http status code: %v", statusCode) - } - return -} - -func queryGraphQL(pat string, query string, result interface{}) error { - queryBytes, err := json.Marshal(&struct { - Query string `json:"query"` - }{query}) - if err != nil { - return err - } - - httpRequest, err := http.NewRequest("POST", "https://api.github.com/graphql", bytes.NewReader(queryBytes)) - if err != nil { - return err - } - httpRequest.SetBasicAuth("", pat) - - return sendJSONRequestSuccessful(httpRequest, result) -} - -func mutateGraphQL(pat string, query string) error { - // Queries and mutations use the same API. But with a mutation, the results aren't useful to us. - return queryGraphQL(pat, query, &struct{}{}) -} - -func findExistingPR(b *branch, githubUser string, originOwner string, githubPAT string) (string, error) { - prQuery := `{ - user(login: "` + githubUser + `") { - pullRequests(states: OPEN, baseRefName: "` + b.mergeTarget + `", first: 5) { - nodes { - title - id - headRepositoryOwner { - login - } - baseRepository { - owner { - login - } - } - } - } - } - }` - // Output structure from the query. We pull out some data to make sure our search result is what - // we expect and avoid relying solely on the search engine query. This may be expanded in the - // future to search for a specific PR among the search results, if necessary. (Needed if we want - // to submit multiple, similar PRs from this bot.) - // - // Declared adjacent to the query because the query determines the structure. - result := &struct { - // Note: Go encoding/json only detects exported properties (capitalized), but it does handle - // matching it to the lowercase JSON for us. - Data struct { - User struct { - PullRequests struct { - Nodes []struct { - Title string - ID string - HeadRepositoryOwner struct { - Login string - } - BaseRepository struct { - Owner struct { - Login string - } - } - } - PageInfo struct { - HasNextPage bool - } - } - } - } - }{} - - if err := queryGraphQL(githubPAT, prQuery, result); err != nil { return "", err } - fmt.Printf("%+v\n", result) - - // Basic search result validation. We could be more flexible in some cases, but the goal here is - // to detect an unknown state early so we don't end up doing something strange. - - if prNodes := len(result.Data.User.PullRequests.Nodes); prNodes != 1 { - return "", fmt.Errorf("expected 1 PR search result, found %v", prNodes) - } - if result.Data.User.PullRequests.PageInfo.HasNextPage { - return "", fmt.Errorf("expected 1 PR search result, but the results say there's another page") - } - - n := result.Data.User.PullRequests.Nodes[0] - if headOwner := n.HeadRepositoryOwner.Login; headOwner != githubUser { - return "", fmt.Errorf("pull request head owner is %v, expected %v", headOwner, githubUser) - } - if baseOwner := n.BaseRepository.Owner.Login; baseOwner != originOwner { - return "", fmt.Errorf("pull request base owner is %v, expected %v", baseOwner, originOwner) - } - - return n.ID, nil + return string(out), nil } diff --git a/eng/go.mod b/eng/go.mod new file mode 100644 index 00000000..f2dad935 --- /dev/null +++ b/eng/go.mod @@ -0,0 +1,7 @@ +// This empty go.mod file causes the directory's contents to be excluded from go commands run in the +// root of the go-infra repository. This approach is from this comment: +// https://github.com/golang/go/issues/30058#issuecomment-543815369 +// +// Exclusion is useful because the "sync" utility clones repositories into "eng/artifacts", so +// running "go build ./..." or other similar commnads would find ".go" files inside the +// sub-repository. By excluding the directory, this is avoided. diff --git a/eng/pipelines/go-update.yml b/eng/pipelines/go-update.yml index 40cee1ab..7fb024b1 100644 --- a/eng/pipelines/go-update.yml +++ b/eng/pipelines/go-update.yml @@ -42,14 +42,10 @@ jobs: - download: build artifact: BuildAssets - - task: GoTool@0 - inputs: - version: 1.17.1 + - template: steps/init-go.yml + - template: steps/set-bot-git-author.yml - pwsh: | - git config --global user.name 'microsoft-golang-bot' - git config --global user.email 'microsoft-golang-bot@users.noreply.github.com' - go run ./cmd/dockerupdatepr ` -branch microsoft/main ` -origin https://microsoft-golang-bot:$(BotAccount-microsoft-golang-bot-PAT)@github.com/microsoft/go-docker ` diff --git a/eng/pipelines/steps/init-go.yml b/eng/pipelines/steps/init-go.yml new file mode 100644 index 00000000..9ece3518 --- /dev/null +++ b/eng/pipelines/steps/init-go.yml @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Set up Go tool in the build machine. This is in a template to make version updates easy. + +steps: + - task: GoTool@0 + inputs: + version: 1.17.2 diff --git a/eng/pipelines/steps/set-bot-git-author.yml b/eng/pipelines/steps/set-bot-git-author.yml new file mode 100644 index 00000000..72b0050c --- /dev/null +++ b/eng/pipelines/steps/set-bot-git-author.yml @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Set up the Git author, in preparation to create a PR. + +steps: + - script: | + git config --global user.name 'microsoft-golang-bot' + git config --global user.email 'microsoft-golang-bot@users.noreply.github.com' + displayName: Set up Git author diff --git a/eng/pipelines/sync-pipeline.yml b/eng/pipelines/sync-pipeline.yml new file mode 100644 index 00000000..4d59516d --- /dev/null +++ b/eng/pipelines/sync-pipeline.yml @@ -0,0 +1,36 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +trigger: none +pr: none +schedules: + - cron: '0 16 * * Mon,Wed,Fri' + displayName: Sync from upstream three times a week + branches: + include: + - microsoft/main + always: true + +variables: + - group: Microsoft-GoLang-bot + +jobs: + - job: Sync + pool: + vmImage: ubuntu-20.04 + workspace: + clean: all + steps: + - template: steps/checkout-unix-task.yml + - template: steps/init-pwsh-task.yml + + - template: steps/init-go.yml + - template: steps/set-bot-git-author.yml + + - pwsh: | + go run ./cmd/sync ` + -git-auth-pat ` + -github-user microsoft-golang-bot ` + -github-pat $(BotAccount-microsoft-golang-bot-PAT) ` + -github-pat-reviewer $(BotAccount-microsoft-golang-review-bot-PAT) + displayName: Sync diff --git a/eng/sync-config.json b/eng/sync-config.json new file mode 100644 index 00000000..ed6ff988 --- /dev/null +++ b/eng/sync-config.json @@ -0,0 +1,36 @@ +[ + { + "Upstream": "https://go.googlesource.com/go", + "Target": "https://github.com/microsoft/go", + "SourceBranches": [ + "master", + "release-branch.go1.15", + "release-branch.go1.16", + "release-branch.go1.17" + ], + "AutoResolveOurs": [ + ".gitattributes", + ".github", + "CODE_OF_CONDUCT.md", + "README.md", + "SECURITY.md", + "SUPPORT.md" + ] + }, + { + "Upstream": "https://github.com/docker-library/golang", + "Target": "https://github.com/microsoft/go-docker", + "SourceBranches": [ + "master" + ], + "AutoResolveOurs": [ + ".gitattributes", + ".github", + ".gitignore", + "CODE_OF_CONDUCT.md", + "README.md", + "SECURITY.md", + "SUPPORT.md" + ] + } +] diff --git a/gitpr/gitpr.go b/gitpr/gitpr.go index fa839d61..cc7c4467 100644 --- a/gitpr/gitpr.go +++ b/gitpr/gitpr.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "log" "net/http" "strings" "time" @@ -17,50 +18,76 @@ var client = http.Client{ Timeout: time.Second * 30, } -// PRBranch contains information about the specific branch to update. During the sync process, -// more info can be added to this struct to be used later. This struct has methods that help -// calculate derived information such as Git ref names. -type PRBranch struct { - // Name of the branch to update, without "refs/heads/". +// PRRefSet contains information about an automatic PR branch and calculates the set of refs that +// would correspond to that PR. +type PRRefSet struct { + // Name of the base branch to update. Do not include "refs/heads/". Name string // Purpose of the PR. This is used to generate the PR branch name, "dev/{Purpose}/{Name}". Purpose string } -func (b PRBranch) PRBranch() string { +// PRBranch is the name of the "head" branch name for this PR, under "dev/{Purpose}/{Name}" +// convention, without the "refs/heads/" prefix. +func (b PRRefSet) PRBranch() string { return "dev/" + b.Purpose + "/" + b.Name } -func (b PRBranch) BaseBranchFetchRefspec() string { - return "refs/heads/" + b.Name + ":refs/heads/" + b.PRBranch() +// BaseBranchFetchRefspec is the refspec with src: PR base branch src, dst: PR head branch dst. This +// can be used with "fetch" to create a fresh dev branch. +func (b PRRefSet) BaseBranchFetchRefspec() string { + return createRefspec(b.Name, b.PRBranch()) } -func (b PRBranch) PRBranchFetchRefspec() string { - return "refs/heads/" + b.PRBranch() + ":refs/heads/" + b.PRBranch() -} - -func (b PRBranch) PRPushRefspec() string { - return b.PRBranch() + ":refs/heads/" + b.PRBranch() +// PRBranchRefspec is the refspec that syncs the dev branch between two repos. +func (b PRRefSet) PRBranchRefspec() string { + return createRefspec(b.PRBranch(), b.PRBranch()) } // CreateGitHubPR creates the data model that can be sent to GitHub to create a PR for this branch. -func (b PRBranch) CreateGitHubPR(headOwner string) GitHubRequest { - return GitHubRequest{ +func (b PRRefSet) CreateGitHubPR(headOwner, title, body string) *GitHubRequest { + return &GitHubRequest{ Head: headOwner + ":" + b.PRBranch(), Base: b.Name, - Title: fmt.Sprintf("Update dependencies in `%v`", b.Name), - Body: fmt.Sprintf( - "🔃 This is an automatically generated PR updating the version of Go in `%v`.\n\n"+ - "This PR should auto-merge itself when PR validation passes.\n\n", - b.Name, - ), + Title: title, + Body: body, MaintainerCanModify: true, Draft: false, } } +// SyncPRRefSet calculates the set of refs that correspond to a PR branch that is performing a Git +// sync from an upstream repository. +type SyncPRRefSet struct { + // UpstreamName is the name of the upstream branch being synced from. + UpstreamName string + PRRefSet +} + +// NewSyncPRRefSet creates a SyncPRRefSet based on the name of an upstream branch. Mapping from +// upstream branch name to "microsoft/"-prefixed branch name happens here. +func NewSyncPRRefSet(upstreamName string) *SyncPRRefSet { + return &SyncPRRefSet{ + upstreamName, + PRRefSet{ + Name: "microsoft/" + strings.ReplaceAll(upstreamName, "master", "main"), + Purpose: "auto-merge", + }, + } +} + +// UpstreamLocalBranch is the name of the upstream ref after it has been fetched locally. +func (b SyncPRRefSet) UpstreamLocalBranch() string { + return "fetched-upstream/" + b.UpstreamName +} + +// UpstreamFetchRefspec fetches the current upstream ref into the local branch. +func (b SyncPRRefSet) UpstreamFetchRefspec() string { + return createRefspec(b.UpstreamName, b.UpstreamLocalBranch()) +} + // Remote is a parsed version of a Git Remote. It helps determine how to send a GitHub PR. type Remote struct { url string @@ -102,7 +129,7 @@ func (r Remote) GetOwnerSlashRepo() string { func GetUsername(pat string) string { request, err := http.NewRequest("GET", "https://api.github.com/user", nil) if err != nil { - panic(err) + log.Panic(err) } request.SetBasicAuth("", pat) @@ -111,7 +138,7 @@ func GetUsername(pat string) string { }{} if err := sendJSONRequestSuccessful(request, response); err != nil { - panic(err) + log.Panic(err) } return response.Login @@ -190,7 +217,7 @@ type GitHubRequestError struct { Message string `json:"message"` } -func PostGitHub(ownerRepo string, request GitHubRequest, pat string) (response *GitHubResponse, err error) { +func PostGitHub(ownerRepo string, request *GitHubRequest, pat string) (response *GitHubResponse, err error) { prSubmitContent, err := json.MarshalIndent(request, "", "") fmt.Printf("Submitting payload: %s\n", prSubmitContent) @@ -274,10 +301,10 @@ func MutateGraphQL(pat string, query string, variables map[string]interface{}) e // FindExistingPR looks for a PR submitted to a target branch with a set of filters. Returns the // result's graphql identity if one match is found, empty string if no matches are found, and an // error if more than one match was found. -func FindExistingPR(b *PRBranch, githubUser string, headOwner string, originOwner string, githubPAT string) (string, error) { - prQuery := `query ($githubUser: String!, $baseRefName: String!) { +func FindExistingPR(r *GitHubRequest, head, target *Remote, headBranch, submitterUser, githubPAT string) (string, error) { + prQuery := `query ($githubUser: String!, $headRefName: String!, $baseRefName: String!) { user(login: $githubUser) { - pullRequests(states: OPEN, baseRefName: $baseRefName, first: 5) { + pullRequests(states: OPEN, headRefName: $headRefName, baseRefName: $baseRefName, first: 5) { nodes { title id @@ -294,8 +321,9 @@ func FindExistingPR(b *PRBranch, githubUser string, headOwner string, originOwne } }` variables := map[string]interface{}{ - "githubUser": githubUser, - "baseRefName": b.Name, + "githubUser": submitterUser, + "headRefName": headBranch, + "baseRefName": r.Base, } // Output structure from the query. We pull out some data to make sure our search result is what // we expect and avoid relying solely on the search engine query. This may be expanded in the @@ -349,11 +377,42 @@ func FindExistingPR(b *PRBranch, githubUser string, headOwner string, originOwne } n := result.Data.User.PullRequests.Nodes[0] - if foundHeadOwner := n.HeadRepositoryOwner.Login; foundHeadOwner != headOwner { - return "", fmt.Errorf("pull request head owner is %v, expected %v", foundHeadOwner, headOwner) + if foundHeadOwner := n.HeadRepositoryOwner.Login; foundHeadOwner != head.GetOwner() { + return "", fmt.Errorf("pull request head owner is %v, expected %v", foundHeadOwner, head.GetOwner()) } - if foundBaseOwner := n.BaseRepository.Owner.Login; foundBaseOwner != originOwner { - return "", fmt.Errorf("pull request base owner is %v, expected %v", foundBaseOwner, originOwner) + if foundBaseOwner := n.BaseRepository.Owner.Login; foundBaseOwner != target.GetOwner() { + return "", fmt.Errorf("pull request base owner is %v, expected %v", foundBaseOwner, target.GetOwner()) } return n.ID, nil } + +// ApprovePR adds an approving review on the target GraphQL PR node ID. The review author is the user +// associated with the PAT. +func ApprovePR(nodeID string, pat string) error { + return MutateGraphQL( + pat, + `mutation ($nodeID: ID!) { + addPullRequestReview(input: {pullRequestId: $nodeID, event: APPROVE, body: "Thanks! Auto-approving."}) { + clientMutationId + } + }`, + map[string]interface{}{"nodeID": nodeID}) +} + +// EnablePRAutoMerge enables PR automerge on the target GraphQL PR node ID. +func EnablePRAutoMerge(nodeID string, pat string) error { + return MutateGraphQL( + pat, + `mutation ($nodeID: ID!) { + enablePullRequestAutoMerge(input: {pullRequestId: $nodeID, mergeMethod: MERGE}) { + clientMutationId + } + }`, + map[string]interface{}{"nodeID": nodeID}) +} + +// createRefspec makes a refspec that will fetch or push a branch "source" to "dest". The args must +// not already have a "refs/heads/" prefix. +func createRefspec(source, dest string) string { + return fmt.Sprintf("refs/heads/%v:refs/heads/%v", source, dest) +}