Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimize release notes generation to use GitHub Milestones #13398

Merged
merged 2 commits into from
Jun 29, 2023
Merged

Optimize release notes generation to use GitHub Milestones #13398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
61e117b
Optimize release notes generation to use GitHub Milestones
frouioui Jun 28, 2023
3188271
Remove threading flag
frouioui Jun 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 35 additions & 201 deletions go/tools/release-notes/release_notes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"regexp"
"sort"
"strings"
"sync"
"text/template"

"github.com/spf13/pflag"
Expand All @@ -40,24 +39,24 @@ type (

labels []label

author struct {
Login string `json:"login"`
pullRequestAuthor struct {
Login string
}

prInfo struct {
Labels labels `json:"labels"`
Number int `json:"number"`
Title string `json:"title"`
Author author `json:"author"`
pullRequestInformation struct {
Number int
Title string
Labels labels
Author pullRequestAuthor
}

prsByComponent = map[string][]prInfo
prsByComponent = map[string][]pullRequestInformation

prsByType = map[string]prsByComponent

sortedPRComponent struct {
Name string
PrInfos []prInfo
PrInfos []pullRequestInformation
}

sortedPRType struct {
Expand All @@ -82,7 +81,6 @@ type (

var (
releaseNotesPath = `changelog/`
numberOfThreads = 10
)

const (
Expand Down Expand Up @@ -188,61 +186,27 @@ func loadKnownIssues(release string) ([]knownIssue, error) {
return knownIssues, nil
}

func loadMergedPRs(from, to string) (prs []string, authors []string, commitCount int, err error) {
// load the git log with "author \t title \t parents"
out, err := execCmd("git", "log", `--pretty=format:%ae%x09%s%x09%P%x09%h`, fmt.Sprintf("%s..%s", from, to))

func loadMergedPRsAndAuthors(name string) (pris []pullRequestInformation, authors []string, err error) {
out, err := execCmd("gh", "pr", "list", "-s", "merged", "-S", fmt.Sprintf("milestone:%s", name), "--json", "number,title,labels,author", "--limit", "5000")
if err != nil {
return
}

return parseGitLog(string(out))
}

func parseGitLog(s string) (prs []string, authorCommits []string, commitCount int, err error) {
rx := regexp.MustCompile(`(.+)\t(.+)\t(.+)\t(.+)`)
mergePR := regexp.MustCompile(`Merge pull request #(\d+)`)
squashPR := regexp.MustCompile(`\(#(\d+)\)`)
authMap := map[string]string{} // here we will store email <-> gh user mappings
lines := strings.Split(s, "\n")
for _, line := range lines {
lineInfo := rx.FindStringSubmatch(line)
if len(lineInfo) != 5 {
log.Fatalf("failed to parse the output from git log: %s", line)
}
authorEmail := lineInfo[1]
title := lineInfo[2]
parents := lineInfo[3]
sha := lineInfo[4]
merged := mergePR.FindStringSubmatch(title)
if len(merged) == 2 {
// this is a merged PR. remember the PR #
prs = append(prs, merged[1])
continue
}

if len(parents) <= lengthOfSingleSHA {
// we have a single parent, and the commit counts
commitCount++
if _, exists := authMap[authorEmail]; !exists {
authMap[authorEmail] = sha
}
}

squashed := squashPR.FindStringSubmatch(title)
if len(squashed) == 2 {
// this is a merged PR. remember the PR #
prs = append(prs, squashed[1])
continue
}
err = json.Unmarshal(out, &pris)
if err != nil {
return nil, nil, err
}

for _, author := range authMap {
authorCommits = append(authorCommits, author)
// Get the full list of distinct PRs authors and sort them
authorMap := map[string]bool{}
for _, pri := range pris {
login := pri.Author.Login
if ok := authorMap[login]; !ok {
authors = append(authors, login)
authorMap[login] = true
}
}

sort.Strings(prs)
sort.Strings(authorCommits) // not really needed, but makes testing easier
sort.Strings(authors)

return
}
Expand All @@ -262,133 +226,10 @@ func execCmd(name string, arg ...string) ([]byte, error) {
return out, nil
}

func loadPRInfo(pr string) (prInfo, error) {
out, err := execCmd("gh", "pr", "view", pr, "--json", "title,number,labels,author")
if err != nil {
return prInfo{}, err
}
var prInfo prInfo
err = json.Unmarshal(out, &prInfo)
return prInfo, err
}

func loadAuthorInfo(sha string) (string, error) {
out, err := execCmd("gh", "api", "/repos/vitessio/vitess/commits/"+sha)
if err != nil {
return "", err
}
var prInfo prInfo
err = json.Unmarshal(out, &prInfo)
if err != nil {
return "", err
}
return prInfo.Author.Login, nil
}

type req struct {
isPR bool
key string
}

func loadAllPRs(prs, authorCommits []string) ([]prInfo, []string, error) {
errChan := make(chan error)
wgDone := make(chan bool)
prChan := make(chan req, len(prs)+len(authorCommits))
// fill the work queue
for _, s := range prs {
prChan <- req{isPR: true, key: s}
}
for _, s := range authorCommits {
prChan <- req{isPR: false, key: s}
}
close(prChan)

var prInfos []prInfo
var authors []string
fmt.Printf("Found %d merged PRs. Loading PR info", len(prs))
wg := sync.WaitGroup{}
mu := sync.Mutex{}

shouldLoad := func(in string) bool {
if in == "" {
return false
}
mu.Lock()
defer mu.Unlock()

for _, existing := range authors {
if existing == in {
return false
}
}
return true
}
addAuthor := func(in string) {
mu.Lock()
defer mu.Unlock()
authors = append(authors, in)
}
addPR := func(in prInfo) {
mu.Lock()
defer mu.Unlock()
prInfos = append(prInfos, in)
}

for i := 0; i < numberOfThreads; i++ {
wg.Add(1)
go func() {
// load meta data about PRs
defer wg.Done()

for b := range prChan {
fmt.Print(".")
if b.isPR {
prInfo, err := loadPRInfo(b.key)
if err != nil {
errChan <- err
break
}
addPR(prInfo)
continue
}
author, err := loadAuthorInfo(b.key)
if err != nil {
errChan <- err
break
}
if shouldLoad(author) {
addAuthor(author)
}

}
}()
}

go func() {
// wait for the loading to finish
wg.Wait()
close(wgDone)
}()

var err error
select {
case <-wgDone:
break
case err = <-errChan:
break
}

fmt.Println()

sort.Strings(authors)

return prInfos, authors, err
}

func groupPRs(prInfos []prInfo) prsByType {
func groupPRs(pris []pullRequestInformation) prsByType {
prPerType := prsByType{}

for _, info := range prInfos {
for _, info := range pris {
var typ, component string
for _, lbl := range info.Labels {
switch {
Expand Down Expand Up @@ -478,11 +319,11 @@ func getStringForKnownIssues(issues []knownIssue) (string, error) {
return buff.String(), nil
}

func groupAndStringifyPullRequest(pr []prInfo) (string, error) {
if len(pr) == 0 {
func groupAndStringifyPullRequest(pris []pullRequestInformation) (string, error) {
if len(pris) == 0 {
return "", nil
}
prPerType := groupPRs(pr)
prPerType := groupPRs(pris)
prStr, err := getStringForPullRequestInfos(prPerType)
if err != nil {
return "", err
Expand All @@ -492,14 +333,10 @@ func groupAndStringifyPullRequest(pr []prInfo) (string, error) {

func main() {
var (
from, versionName, summaryFile string
to = "HEAD"
versionName, summaryFile string
)
pflag.StringVarP(&from, "from", "f", "", "from sha/tag/branch")
pflag.StringVarP(&to, "to", "t", to, "to sha/tag/branch")
pflag.StringVarP(&versionName, "version", "v", "", "name of the version (has to be the following format: v11.0.0)")
pflag.StringVarP(&summaryFile, "summary", "s", "", "readme file on which there is a summary of the release")
pflag.IntVar(&numberOfThreads, "threads", numberOfThreads, "Define the number of threads used to fetch data from GitHub's API. Lower this number if you hit request limit errors.")
pflag.Parse()

// The -version flag must be of a valid format.
Expand Down Expand Up @@ -547,26 +384,23 @@ func main() {
releaseNotes.KnownIssues = knownIssuesStr

// changelog with pull requests
prs, authorCommits, commits, err := loadMergedPRs(from, to)
if err != nil {
log.Fatal(err)
}
prInfos, authors, err := loadAllPRs(prs, authorCommits)
prs, authors, err := loadMergedPRsAndAuthors(versionName)
if err != nil {
log.Fatal(err)
}
releaseNotes.ChangeLog, err = groupAndStringifyPullRequest(prInfos)

releaseNotes.ChangeLog, err = groupAndStringifyPullRequest(prs)
if err != nil {
log.Fatal(err)
}

// changelog metrics
if commits > 0 && len(authors) > 0 {
if len(prs) > 0 && len(authors) > 0 {
releaseNotes.ChangeMetrics = fmt.Sprintf(`
The release includes %d commits (excluding merges)
The release includes %d merged Pull Requests.

Thanks to all our contributors: @%s
`, commits, strings.Join(authors, ", @"))
`, len(prs), strings.Join(authors, ", @"))
}

if err := releaseNotes.generate(nil, nil); err != nil {
Expand Down
Loading